feat:增加fluent日志

2025-09-25 23:33:50 +08:00
parent 5e3dc21ed1
commit a8fdd04c41
4 changed files with 112 additions and 23 deletions
--- a/2
+++ b/2
@@ -1,5 +1,7 @@
 FROM python:3.12-slim
 WORKDIR /app
+RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+RUN echo 'Asia/Shanghai' > /etc/timezone
 COPY ./packages /app/packages
 COPY requirements.txt /app/
 RUN pip install --no-cache-dir --no-index --find-links=/app/packages -r requirements.txt
--- a/config/logging.py
+++ b/config/logging.py
@@ -1,8 +1,15 @@
+import atexit
+
+from fluent import sender
 from loguru import logger
 from pathlib import Path

 from config.setting import settings

+FLUENTD_HOST = '14.103.235.151'
+FLUENTD_PORT = 24422
+TOPIC_TAG = 'blog-service'
+
 # 日志目录
 LOG_DIR = Path(__file__).parent.parent.parent / "logs"
 LOG_DIR.mkdir(exist_ok=True)
@@ -24,6 +31,40 @@ FILE_FORMAT = (
    "{name}:{function}:{line} - {message}"
 )

+fluent_sender = sender.FluentSender(
+    tag=TOPIC_TAG,
+    host=FLUENTD_HOST,
+    port=FLUENTD_PORT,
+    buffer_max_size=8 * 1024 * 1024,
+    timeout=3.0
+)
+
+
+def log_to_fluent(message):
+    try:
+        record = message.record
+
+        # 构建结构化日志数据
+        log_data = {
+            'topic': TOPIC_TAG,
+            'timestamp': record['time'].timestamp(),
+            'level': record['level'].name.lower(),
+            'message': record['message'],
+            'source': f"{record['file'].path}:{record['line']}",
+            'module': record['module'],
+            'function': record['function'],
+            'process_id': record['process'].id,
+            'thread_id': record['thread'].id,
+            **record['extra']
+        }
+
+        if not fluent_sender.emit(TOPIC_TAG, log_data):
+            print(f"Fluentd 发送失败: {fluent_sender.last_error}")
+
+    except Exception as e:
+        print(f"日志处理异常: {str(e)}")
+
+
 # 移除默认处理器
 logger.remove()

@@ -34,32 +75,42 @@ logger.add(
    format=STDOUT_FORMAT,
    colorize=True,
    backtrace=True,  # 显示完整异常堆栈
-    diagnose=True,   # 显示详细异常信息
+    diagnose=True,  # 显示详细异常信息
 )

-# 添加文件处理器 - 常规日志
 logger.add(
-    sink=LOG_DIR / "app.log",
-    level="INFO",
-    format=FILE_FORMAT,
-    rotation="10 MB",        # 日志文件大小达到10MB时自动分割
-    retention="7 days",      # 保留7天的日志
-    compression="zip",       # 归档时压缩为zip
-    enqueue=True,            # 异步写入
-    serialize=False,         # 不使用JSON格式
+    log_to_fluent,
+    level="INFO",  # 处理 INFO 及以上级别
+    format="{message}",  # 原始消息（实际使用结构化数据）
+    backtrace=True,  # 启用堆栈回溯
+    diagnose=True  # 显示诊断信息
 )

-# 添加文件处理器 - 错误日志
-logger.add(
-    sink=LOG_DIR / "error.log",
-    level="ERROR",
-    format=FILE_FORMAT,
-    rotation="10 MB",
-    retention="30 days",
-    compression="zip",
-    enqueue=True,
-    serialize=False,
-)
+# # 添加文件处理器 - 常规日志
+# logger.add(
+#     sink=LOG_DIR / "app.log",
+#     level="INFO",
+#     format=FILE_FORMAT,
+#     rotation="10 MB",  # 日志文件大小达到10MB时自动分割
+#     retention="7 days",  # 保留7天的日志
+#     compression="zip",  # 归档时压缩为zip
+#     enqueue=True,  # 异步写入
+#     serialize=False,  # 不使用JSON格式
+# )
+#
+# # 添加文件处理器 - 错误日志
+# logger.add(
+#     sink=LOG_DIR / "error.log",
+#     level="ERROR",
+#     format=FILE_FORMAT,
+#     rotation="10 MB",
+#     retention="30 days",
+#     compression="zip",
+#     enqueue=True,
+#     serialize=False,
+# )
+
+atexit.register(fluent_sender.close)

 # 导出配置好的logger
-__all__ = ["logger"]
+__all__ = ["logger"]
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1 @@
-fastapi~=0.115.12
+fastapi~=0.115.12
--- a/sys.stdout
+++ b/sys.stdout
@@ -1108,3 +1108,39 @@ category
 [32m2025-09-25 14:59:54.375[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m20[0m - [1m查询参数: {'year': '2025'}[0m
 [32m2025-09-25 15:00:09.535[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/blog/page?currentPage=1&pageSize=10[0m
 [32m2025-09-25 15:00:09.536[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m20[0m - [1m查询参数: {'currentPage': '1', 'pageSize': '10'}[0m
+[32m2025-09-25 22:49:58.584[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/[0m
+[32m2025-09-25 22:50:00.691[0m | [33m[1mWARNING [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m26[0m - [33m[1m响应: 404[0m
+[32m2025-09-25 22:50:02.808[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/docs[0m
+[32m2025-09-25 22:50:09.497[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/openapi.json[0m
+[32m2025-09-25 22:50:16.478[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/blog/page?currentPage=1&pageSize=10[0m
+[32m2025-09-25 22:50:18.553[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m20[0m - [1m查询参数: {'currentPage': '1', 'pageSize': '10'}[0m
+[32m2025-09-25 22:50:20.723[0m | [41m[1mCRITICAL[0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m38[0m - [41m[1m数据库异常: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on 'host.docker.internal' ([Errno 11001] getaddrinfo failed)")
+(Background on this error at: https://sqlalche.me/e/20/e3q8)[0m
+[32m2025-09-25 22:50:51.042[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/blog/page?currentPage=1&pageSize=10[0m
+[32m2025-09-25 22:50:53.126[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m20[0m - [1m查询参数: {'currentPage': '1', 'pageSize': '10'}[0m
+[32m2025-09-25 22:50:55.241[0m | [41m[1mCRITICAL[0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m38[0m - [41m[1m数据库异常: (pymysql.err.OperationalError) (1045, "Access denied for user 'root'@'localhost' (using password: YES)")
+(Background on this error at: https://sqlalche.me/e/20/e3q8)[0m
+[32m2025-09-25 22:51:09.586[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/blog/page?currentPage=1&pageSize=10[0m
+[32m2025-09-25 22:51:11.676[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m20[0m - [1m查询参数: {'currentPage': '1', 'pageSize': '10'}[0m
+[32m2025-09-25 22:52:58.339[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/blog/page?currentPage=1&pageSize=10[0m
+[32m2025-09-25 22:53:01.352[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m20[0m - [1m查询参数: {'currentPage': '1', 'pageSize': '10'}[0m
+[32m2025-09-25 22:54:16.612[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/blog/page?currentPage=1&pageSize=10[0m
+[32m2025-09-25 22:54:16.622[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m20[0m - [1m查询参数: {'currentPage': '1', 'pageSize': '10'}[0m
+[32m2025-09-25 22:57:08.357[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/blog/page?currentPage=1&pageSize=10[0m
+[32m2025-09-25 22:57:08.357[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m20[0m - [1m查询参数: {'currentPage': '1', 'pageSize': '10'}[0m
+[32m2025-09-25 22:57:58.433[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/blog/page?currentPage=1&pageSize=10[0m
+[32m2025-09-25 22:57:58.433[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m20[0m - [1m查询参数: {'currentPage': '1', 'pageSize': '10'}[0m
+[32m2025-09-25 23:03:57.445[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/blog/page?currentPage=1&pageSize=10[0m
+[32m2025-09-25 23:03:57.445[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m20[0m - [1m查询参数: {'currentPage': '1', 'pageSize': '10'}[0m
+[32m2025-09-25 23:04:29.198[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/blog/page?currentPage=1&pageSize=10[0m
+[32m2025-09-25 23:04:29.198[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m20[0m - [1m查询参数: {'currentPage': '1', 'pageSize': '10'}[0m
+[32m2025-09-25 23:11:27.170[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/blog/page?currentPage=1&pageSize=10[0m
+[32m2025-09-25 23:11:27.171[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m20[0m - [1m查询参数: {'currentPage': '1', 'pageSize': '10'}[0m
+[32m2025-09-25 23:17:14.162[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/blog/page?currentPage=1&pageSize=10[0m
+[32m2025-09-25 23:17:14.164[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m20[0m - [1m查询参数: {'currentPage': '1', 'pageSize': '10'}[0m
+[32m2025-09-25 23:18:42.029[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/blog/page?currentPage=1&pageSize=10[0m
+[32m2025-09-25 23:18:42.029[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m20[0m - [1m查询参数: {'currentPage': '1', 'pageSize': '10'}[0m
+[32m2025-09-25 23:20:09.630[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/blog/page?currentPage=1&pageSize=10[0m
+[32m2025-09-25 23:20:09.668[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m20[0m - [1m查询参数: {'currentPage': '1', 'pageSize': '10'}[0m
+[32m2025-09-25 23:22:00.941[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m18[0m - [1m请求: GET http://127.0.0.1:8082/blog/page?currentPage=1&pageSize=10[0m
+[32m2025-09-25 23:22:00.942[0m | [1mINFO    [0m | [36mmiddleware.exceptions[0m:[36mglobal_exception_handler[0m:[36m20[0m - [1m查询参数: {'currentPage': '1', 'pageSize': '10'}[0m