Add granian and celery metrics collection (#1057)

* Add granian and celery metrics collection * Update dashboards * Add asyncio instrument * Fix opera log finally
2026-09-21 13:12:24 +00:00 · 2026-02-03 20:44:05 +08:00
parent 6f1c27786d
commit 646a0ec2fc
19 changed files with 877 additions and 828 deletions
@@ -1096,7 +1096,7 @@
        ]
      },
      "gridPos": {
-        "h": 10,
+        "h": 12,
        "w": 12,
        "x": 12,
        "y": 10
@@ -1259,7 +1259,7 @@
        ]
      },
      "gridPos": {
-        "h": 10,
+        "h": 12,
        "w": 12,
        "x": 0,
        "y": 10
@@ -1396,7 +1396,7 @@
        "overrides": []
      },
      "gridPos": {
-        "h": 10,
+        "h": 12,
        "w": 12,
        "x": 0,
        "y": 19
@@ -1494,7 +1494,7 @@
        "overrides": []
      },
      "gridPos": {
-        "h": 10,
+        "h": 12,
        "w": 12,
        "x": 12,
        "y": 19
@@ -1651,7 +1651,7 @@
        "overrides": []
      },
      "gridPos": {
-        "h": 8,
+        "h": 12,
        "w": 8,
        "x": 8,
        "y": 28
@@ -1719,7 +1719,7 @@
        "overrides": []
      },
      "gridPos": {
-        "h": 8,
+        "h": 12,
        "w": 8,
        "x": 16,
        "y": 28
@@ -1835,7 +1835,7 @@
        "overrides": []
      },
      "gridPos": {
-        "h": 10,
+        "h": 12,
        "w": 12,
        "x": 0,
        "y": 37
@@ -1981,7 +1981,7 @@
        ]
      },
      "gridPos": {
-        "h": 10,
+        "h": 12,
        "w": 12,
        "x": 12,
        "y": 37
@@ -2027,6 +2027,202 @@
      ],
      "title": "任务吞吐量 (每分钟)",
      "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 47
+      },
+      "id": 106,
+      "title": "🔍 链路追踪",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "tempo",
+        "uid": "tempo"
+      },
+      "description": "Celery Worker 的分布式追踪数据，点击 Trace ID 可跳转到 Explore 查看完整调用链",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "left",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "filterable": false,
+            "inspect": false
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Service"
+            },
+            "properties": [
+              {
+                "id": "custom.hidden",
+                "value": true
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Trace ID"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 350
+              },
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "type": "color-text"
+                }
+              },
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#6E9FFF",
+                  "mode": "fixed"
+                }
+              },
+              {
+                "id": "links",
+                "value": [
+                  {
+                    "title": "查看调用链",
+                    "url": "/explore?orgId=1&left={\"datasource\":\"tempo\",\"queries\":[{\"refId\":\"A\",\"queryType\":\"traceql\",\"query\":\"${__value.raw}\"}],\"range\":{\"from\":\"now-1h\",\"to\":\"now\"}}"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Start time"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 200
+              },
+              {
+                "id": "displayName",
+                "value": "时间"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Name"
+            },
+            "properties": [
+              {
+                "id": "displayName",
+                "value": "任务"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Duration"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 120
+              },
+              {
+                "id": "displayName",
+                "value": "耗时"
+              },
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "type": "gauge",
+                  "mode": "gradient",
+                  "valueDisplayMode": "text"
+                }
+              },
+              {
+                "id": "color",
+                "value": {
+                  "mode": "continuous-GrYlRd"
+                }
+              },
+              {
+                "id": "max",
+                "value": 10000
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 48
+      },
+      "id": 60,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false
+        }
+      },
+      "transformations": [
+        {
+          "id": "convertFieldType",
+          "options": {
+            "conversions": [
+              {
+                "targetField": "Start time",
+                "destinationType": "time"
+              }
+            ]
+          }
+        },
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": {},
+            "sort": [
+              {
+                "field": "Start time",
+                "desc": true
+              }
+            ]
+          }
+        }
+      ],
+      "pluginVersion": "12.0.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "tempo",
+            "uid": "tempo"
+          },
+          "queryType": "traceql",
+          "query": "{resource.service.name=\"fba_celery_worker\" && name!=\"connect\"}",
+          "limit": 20,
+          "refId": "A",
+          "tableType": "traces"
+        }
+      ],
+      "title": "Worker Traces",
+      "type": "table"
    }
  ],
  "preload": false,
@@ -111,7 +111,7 @@
            "uid": "prometheus"
          },
          "editorMode": "code",
-          "expr": "sum(fba_request_total{app_name=\"$app_name\", path!=\"/metrics\"})",
+          "expr": "sum(fba_request_total{app_name=\"fba_server\", path!=\"/metrics\"})",
          "instant": true,
          "legendFormat": "",
          "refId": "A"
@@ -186,7 +186,7 @@
            "uid": "prometheus"
          },
          "editorMode": "code",
-          "expr": "sum(rate(fba_request_total{app_name=\"$app_name\", path!=\"/metrics\"}[5m]))",
+          "expr": "sum(rate(fba_request_total{app_name=\"fba_server\", path!=\"/metrics\"}[5m]))",
          "instant": true,
          "legendFormat": "",
          "refId": "A"
@@ -262,7 +262,7 @@
            "uid": "prometheus"
          },
          "editorMode": "code",
-          "expr": "sum(fba_response_total{app_name=\"$app_name\", status_code=~\"2.*\", path!=\"/metrics\"}) / sum(fba_response_total{app_name=\"$app_name\", path!=\"/metrics\"})",
+          "expr": "sum(fba_response_total{app_name=\"fba_server\", status_code=~\"2.*\", path!=\"/metrics\"}) / sum(fba_response_total{app_name=\"fba_server\", path!=\"/metrics\"})",
          "instant": true,
          "legendFormat": "",
          "refId": "A"
@@ -335,7 +335,7 @@
            "uid": "prometheus"
          },
          "editorMode": "code",
-          "expr": "sum(fba_exception_total{app_name=\"$app_name\"}) or vector(0)",
+          "expr": "sum(fba_exception_total{app_name=\"fba_server\"}) or vector(0)",
          "instant": true,
          "legendFormat": "",
          "refId": "A"
@@ -410,7 +410,7 @@
            "uid": "prometheus"
          },
          "editorMode": "code",
-          "expr": "sum(fba_request_cost_time_sum{app_name=\"$app_name\", path!=\"/metrics\"}) / sum(fba_request_cost_time_count{app_name=\"$app_name\", path!=\"/metrics\"})",
+          "expr": "sum(fba_request_cost_time_sum{app_name=\"fba_server\", path!=\"/metrics\"}) / sum(fba_request_cost_time_count{app_name=\"fba_server\", path!=\"/metrics\"})",
          "legendFormat": "",
          "refId": "A"
        }
@@ -482,7 +482,7 @@
            "uid": "prometheus"
          },
          "editorMode": "code",
-          "expr": "sum(fba_request_in_progress{app_name=\"$app_name\", path!=\"/metrics\"}) or vector(0)",
+          "expr": "sum(fba_request_in_progress{app_name=\"fba_server\", path!=\"/metrics\"}) or vector(0)",
          "instant": true,
          "legendFormat": "",
          "refId": "A"
@@ -562,7 +562,7 @@
        "overrides": []
      },
      "gridPos": {
-        "h": 10,
+        "h": 12,
        "w": 12,
        "x": 0,
        "y": 6
@@ -593,7 +593,7 @@
            "uid": "prometheus"
          },
          "editorMode": "code",
-          "expr": "rate(fba_request_total{app_name=\"$app_name\", path!=\"/metrics\"}[5m])",
+          "expr": "rate(fba_request_total{app_name=\"fba_server\", path!=\"/metrics\"}[5m])",
          "legendFormat": "{{method}} {{path}}",
          "refId": "A"
        }
@@ -660,7 +660,7 @@
        "overrides": []
      },
      "gridPos": {
-        "h": 10,
+        "h": 12,
        "w": 12,
        "x": 12,
        "y": 6
@@ -691,7 +691,7 @@
            "uid": "prometheus"
          },
          "editorMode": "code",
-          "expr": "fba_request_in_progress{app_name=\"$app_name\", path!=\"/metrics\"}",
+          "expr": "fba_request_in_progress{app_name=\"fba_server\", path!=\"/metrics\"}",
          "legendFormat": "{{method}} {{path}}",
          "refId": "A"
        }
@@ -780,7 +780,7 @@
        "overrides": []
      },
      "gridPos": {
-        "h": 10,
+        "h": 12,
        "w": 8,
        "x": 0,
        "y": 17
@@ -810,7 +810,7 @@
            "uid": "prometheus"
          },
          "editorMode": "code",
-          "expr": "sum by(path) (fba_response_total{app_name=\"$app_name\", status_code=~\"2.*\", path!=\"/metrics\"}) / sum by(path) (fba_response_total{app_name=\"$app_name\", path!=\"/metrics\"})",
+          "expr": "sum by(path) (fba_response_total{app_name=\"fba_server\", status_code=~\"2.*\", path!=\"/metrics\"}) / sum by(path) (fba_response_total{app_name=\"fba_server\", path!=\"/metrics\"})",
          "legendFormat": "{{path}}",
          "refId": "A"
        }
@@ -887,7 +887,7 @@
        "overrides": []
      },
      "gridPos": {
-        "h": 10,
+        "h": 12,
        "w": 8,
        "x": 8,
        "y": 17
@@ -917,7 +917,7 @@
            "uid": "prometheus"
          },
          "editorMode": "code",
-          "expr": "sum by(path) (fba_response_total{app_name=\"$app_name\", status_code=~\"5.*\", path!=\"/metrics\"}) / sum by(path) (fba_response_total{app_name=\"$app_name\", path!=\"/metrics\"})",
+          "expr": "sum by(path) (fba_response_total{app_name=\"fba_server\", status_code=~\"5.*\", path!=\"/metrics\"}) / sum by(path) (fba_response_total{app_name=\"fba_server\", path!=\"/metrics\"})",
          "legendFormat": "{{path}}",
          "refId": "A"
        }
@@ -948,7 +948,7 @@
        "overrides": []
      },
      "gridPos": {
-        "h": 10,
+        "h": 12,
        "w": 8,
        "x": 16,
        "y": 17
@@ -985,7 +985,7 @@
            "uid": "prometheus"
          },
          "editorMode": "code",
-          "expr": "sum by(path) (fba_request_total{app_name=\"$app_name\", path!=\"/metrics\"})",
+          "expr": "sum by(path) (fba_request_total{app_name=\"fba_server\", path!=\"/metrics\"})",
          "legendFormat": "{{path}}",
          "refId": "A"
        }
@@ -1124,7 +1124,7 @@
            "uid": "prometheus"
          },
          "editorMode": "code",
-          "expr": "fba_request_cost_time_sum{app_name=\"$app_name\", path!=\"/metrics\"} / fba_request_cost_time_count{app_name=\"$app_name\", path!=\"/metrics\"}",
+          "expr": "fba_request_cost_time_sum{app_name=\"fba_server\", path!=\"/metrics\"} / fba_request_cost_time_count{app_name=\"fba_server\", path!=\"/metrics\"}",
          "format": "table",
          "instant": true,
          "legendFormat": "__auto",
@@ -1221,7 +1221,7 @@
        "overrides": []
      },
      "gridPos": {
-        "h": 10,
+        "h": 12,
        "w": 12,
        "x": 0,
        "y": 26
@@ -1251,7 +1251,7 @@
            "uid": "prometheus"
          },
          "editorMode": "code",
-          "expr": "histogram_quantile(0.99, sum(rate(fba_request_cost_time_bucket{app_name=\"$app_name\", path!=\"/metrics\"}[5m])) by(path, le))",
+          "expr": "histogram_quantile(0.99, sum(rate(fba_request_cost_time_bucket{app_name=\"fba_server\", path!=\"/metrics\"}[5m])) by(path, le))",
          "legendFormat": "{{path}}",
          "refId": "A"
        }
@@ -1326,7 +1326,7 @@
        "overrides": []
      },
      "gridPos": {
-        "h": 10,
+        "h": 12,
        "w": 12,
        "x": 12,
        "y": 26
@@ -1356,7 +1356,7 @@
            "uid": "prometheus"
          },
          "editorMode": "code",
-          "expr": "histogram_quantile(0.95, sum(rate(fba_request_cost_time_bucket{app_name=\"$app_name\", path!=\"/metrics\"}[5m])) by(path, le))",
+          "expr": "histogram_quantile(0.95, sum(rate(fba_request_cost_time_bucket{app_name=\"fba_server\", path!=\"/metrics\"}[5m])) by(path, le))",
          "legendFormat": "{{path}}",
          "refId": "A"
        }
@@ -1735,7 +1735,7 @@
            "uid": "loki"
          },
          "editorMode": "code",
-          "expr": "(rate({service_name=\"$app_name\"} |= \"$trace_id\" [1m]))",
+          "expr": "(rate({service_name=\"fba_server\"} | json | target != \"/metrics\" |= \"$trace_id\" [1m]))",
          "legendFormat": "{{level}}",
          "refId": "A"
        }
@@ -1779,7 +1779,7 @@
            "uid": "loki"
          },
          "editorMode": "code",
-          "expr": "{service_name=\"$app_name\"} | json | line_format \"{{.body}}\" |= \"$trace_id\"",
+          "expr": "{service_name=\"fba_server\"} | json | target != \"/metrics\" | line_format \"{{.body}}\" |= \"$trace_id\"",
          "refId": "A"
        }
      ],
@@ -1797,7 +1797,7 @@
        "overrides": []
      },
      "gridPos": {
-        "h": 8,
+        "h": 12,
        "w": 24,
        "x": 0,
        "y": 58
@@ -1822,7 +1822,7 @@
            "uid": "loki"
          },
          "editorMode": "code",
-          "expr": "{service_name=\"$app_name\"} | json | level=~\"ERROR|CRITICAL\" | line_format \"{{.body}}\" |= \"$trace_id\"",
+          "expr": "{service_name=\"fba_server\"} | json | target != \"/metrics\" | level=~\"ERROR|CRITICAL\" | line_format \"{{.body}}\" |= \"$trace_id\"",
          "refId": "A"
        }
      ],
@@ -1839,29 +1839,6 @@
  ],
  "templating": {
    "list": [
-      {
-        "current": {
-          "selected": true,
-          "text": "fba 服务",
-          "value": "fba_server"
-        },
-        "datasource": {
-          "type": "prometheus",
-          "uid": "prometheus"
-        },
-        "definition": "label_values(fba_app_info, app_name)",
-        "description": "数据来自哪个应用",
-        "label": "应用名称",
-        "name": "app_name",
-        "query": {
-          "qryType": 1,
-          "query": "label_values(fba_app_info, app_name)",
-          "refId": "PrometheusVariableQueryEditor-VariableQuery"
-        },
-        "refresh": 1,
-        "regex": "",
-        "type": "query"
-      },
      {
        "current": {
          "text": "",
@@ -11,28 +11,16 @@
 // ============================================================================

 // OTLP 接收器配置
-// OpenTelemetry Protocol (OTLP) 是 OpenTelemetry 的标准数据传输协议
 // 支持同时接收日志、指标和链路追踪数据
 otelcol.receiver.otlp "default" {
-  // gRPC 协议配置
-  // gRPC 是高性能的 RPC 框架，适用于服务间通信
  grpc {
    endpoint = "0.0.0.0:4317"
-    // 0.0.0.0 表示监听所有网络接口
-    // 4317 是 OTLP gRPC 的标准端口
-    // 应用程序通过此端口发送遥测数据
  }

-  // HTTP 协议配置
-  // HTTP 协议更通用，适用于无法使用 gRPC 的场景
  http {
    endpoint = "0.0.0.0:4318"
-    // 4318 是 OTLP HTTP 的标准端口
-    // 支持 JSON 和 Protobuf 格式的数据
  }

-  // 输出配置
-  // 定义接收到的数据发送到哪些处理器
  output {
    logs    = [otelcol.processor.batch.default.input]
    metrics = [otelcol.processor.batch.default.input]
@@ -43,8 +31,6 @@ otelcol.receiver.otlp "default" {
 // 批处理器配置
 // 将多个遥测数据项合并成批次，提高传输效率
 otelcol.processor.batch "default" {
-  // 输出配置
-  // 定义处理后的数据发送到哪些导出器
  output {
    logs    = [otelcol.exporter.loki.default.input]
    metrics = [otelcol.exporter.prometheus.default.input]
@@ -87,14 +73,8 @@ prometheus.scrape "prometheus" {
 		__address__ = "fba_prometheus:9090",
 	}]
 	forward_to      = [prometheus.remote_write.prom_server.receiver]
-	// 抓取的指标发送到远程写入组件
-
 	job_name        = "fba_prometheus"
-	// 任务名称
-	// 将作为 job 标签添加到所有抓取的指标上
-
 	scrape_interval = "15s"
-	// 抓取间隔
 }

 // FastAPI 后端服务指标抓取
@@ -104,17 +84,21 @@ prometheus.scrape "fba_server" {
 		__address__ = "fba_server:8001",
 	}]
 	forward_to      = [prometheus.remote_write.prom_server.receiver]
-	// 抓取的指标发送到远程写入组件
-
 	job_name        = "fba_server"
-	// 任务名称
-	// 用于在 Prometheus 查询中区分不同服务
-
 	scrape_interval = "5s"
-	// 抓取间隔
-
 	scrape_timeout  = "5s"
-	// 抓取超时时间
+}
+
+// Granian 服务器指标抓取
+// 监控 Granian ASGI 服务器的性能指标
+prometheus.scrape "fba_granian" {
+	targets = [{
+		__address__ = "fba_server:8002",
+	}]
+	forward_to      = [prometheus.remote_write.prom_server.receiver]
+	job_name        = "fba_granian"
+	scrape_interval = "5s"
+	scrape_timeout  = "5s"
 }

 // Celery Exporter 指标抓取
@@ -124,17 +108,9 @@ prometheus.scrape "fba_celery_exporter" {
 		__address__ = "fba_celery_exporter:9808",
 	}]
 	forward_to      = [prometheus.remote_write.prom_server.receiver]
-	// 抓取的指标发送到远程写入组件
-
 	job_name        = "fba_celery_exporter"
-	// 任务名称
-	// 用于标识 Celery 相关的指标
-
-	scrape_interval = "5s"
-	// 抓取间隔
-
-	scrape_timeout  = "5s"
-	// 抓取超时时间
+	scrape_interval = "15s"
+	scrape_timeout  = "15s"
 }

 // Tempo 导出器配置
@@ -142,17 +118,9 @@ prometheus.scrape "fba_celery_exporter" {
 otelcol.exporter.otlp "tempo" {
  client {
    endpoint = "fba_tempo:4317"
-
    tls {
        insecure             = true
-        // 禁用 TLS 加密
-        // 在 Docker 内部网络中通信时可以禁用
-        // 生产环境建议启用 TLS
-
        insecure_skip_verify = true
-        // 跳过 TLS 证书验证
-        // 仅在开发/测试环境使用
-        // 生产环境应配置正确的证书
    }
  }
 }
@@ -8,29 +8,12 @@
 # 当前仅支持版本 1
 apiVersion: 1

-# providers: 仪表盘提供者列表
-# 定义 Grafana 从哪里加载仪表盘以及如何管理它们
+# 仪表盘提供者列表
 providers:
  - name: 'fba_server'
-    # 提供者名称，用于标识和日志记录
-
    orgId: 1
-    # 组织 ID
-    # 指定仪表盘属于哪个组织
-    # 默认值: 1（默认组织）
-
    type: 'file'
-    # 提供者类型
-    # 可选值:
-    #   - file: 从本地文件系统加载仪表盘 JSON 文件
-
    disableDeletion: true
-    # 禁止通过 UI 删除仪表盘
-
    editable: false
-    # 是否允许通过 UI 编辑仪表盘
-
    options:
      path: '/etc/grafana/dashboards'
-      # 仪表盘 JSON 文件所在目录（容器内路径）
-      # Grafana 会自动扫描此目录下的所有 .json 文件
@@ -8,8 +8,7 @@
 # 当前仅支持版本 1
 apiVersion: 1

-# datasources: 数据源列表
-# 定义 Grafana 连接的所有后端数据存储
+# 数据源列表
 datasources:
  # ==========================================================================
  # Loki 数据源配置
@@ -17,51 +16,18 @@ datasources:
  # 官方文档: https://grafana.com/docs/loki/latest/
  # ==========================================================================
  - name: Loki
-    # 数据源显示名称，在 Grafana UI 中显示
-
    uid: loki
-    # 数据源唯一标识符
-    # 用于在仪表盘和告警规则中引用此数据源
-
    type: loki
-    # 数据源类型
-    # 必须与 Grafana 支持的数据源插件类型匹配
-
    url: http://fba_loki:3100
-    # Loki 服务器地址
-
    isDefault: false
-    # 是否设为默认数据源
-
    jsonData:
-      # 数据源特定的 JSON 配置
-
      derivedFields:
-        # 派生字段配置
-        # 用于从日志中提取字段并创建链接到其他数据源
-        # 这是实现日志到链路追踪关联的关键配置
-
        - datasourceUid: tempo
-          # 目标数据源的 UID
-          # 点击链接时将跳转到此数据源
-
          matcherRegex: '"traceid":"([a-f0-9]{32})"'
-          # 正则表达式，用于从日志内容中提取 Trace ID
-
          matcherType: regex
-          # 匹配器类型
-          # 可选值: regex（正则表达式）
-
          name: TraceID
-          # 派生字段的名称
-          # 将显示在日志详情中
-
          url: $${__value.raw}
-          # 链接 URL 模板
-          # $${__value.raw} 表示提取的原始值
-
          urlDisplayLabel: 查看 Trace
-          # 链接显示的文本标签

  # ==========================================================================
  # Prometheus 数据源配置
@@ -69,35 +35,15 @@ datasources:
  # 官方文档: https://prometheus.io/docs/
  # ==========================================================================
  - name: Prometheus
-    # 数据源显示名称
-
    uid: prometheus
-    # 数据源唯一标识符
-
    type: prometheus
-    # 数据源类型
-
    url: http://fba_prometheus:9090
-    # Prometheus 服务器地址
-
    isDefault: true
-    # 设为默认数据源
-
    jsonData:
      httpMethod: POST
-      # 查询使用的 HTTP 方法
-
      exemplarTraceIdDestinations:
-        # Exemplar（范例）配置
-        # Exemplar 是指标数据点关联的 Trace ID
-        # 用于从指标跳转到对应的链路追踪
-
        - name: TraceID
-          # Exemplar 中 Trace ID 的字段名称
-
          datasourceUid: tempo
-          # 目标 Tempo 数据源的 UID
-          # 点击 Exemplar 时将跳转到此数据源查看 Trace

  # ==========================================================================
  # Tempo 数据源配置
@@ -105,60 +51,22 @@ datasources:
  # 官方文档: https://grafana.com/docs/tempo/latest/
  # ==========================================================================
  - name: Tempo
-    # 数据源显示名称
-
    uid: tempo
-    # 数据源唯一标识符
-
    type: tempo
-    # 数据源类型
-
    url: http://fba_tempo:3200
-    # Tempo 服务器地址
-
    jsonData:
      nodeGraph:
        enabled: true
-        # 启用节点图功能
-        # 可视化展示服务之间的调用关系图
-
      tracesToLogsV2:
-        # Trace 到日志的关联配置（V2 版本）
-        # 允许从 Trace 详情页面跳转到相关日志
-
        datasourceUid: loki
-        # 日志数据源的 UID
-
        filterByTraceID: true
-        # 是否按 Trace ID 过滤日志
-
        spanStartTimeShift: "-1m"
-        # Span 开始时间偏移
-        # 用于捕获 Span 开始前的相关日志
-
        spanEndTimeShift: "1m"
-        # Span 结束时间偏移
-        # 用于捕获 Span 结束后的相关日志
-
        customQuery: true
-        # 启用自定义查询
-        # 允许使用下面的 query 字段自定义日志查询语句
-
        query: '{service_name="fba_server"} | json | traceid="${__span.traceId}"'
-        # 自定义 LogQL 查询语句
-
      search:
        hide: false
-        # 是否隐藏搜索功能
-
      serviceMap:
        datasourceUid: prometheus
-        # 服务地图数据源
-        # 使用 Prometheus 中的指标数据生成服务依赖图
-        # 需要 Tempo 的 metrics_generator 功能配合
-
      lokiSearch:
        datasourceUid: loki
-        # Loki 搜索数据源
-        # 允许在 Tempo 中使用 Loki 进行日志搜索
-        # 实现基于日志内容查找 Trace 的功能
@@ -4,10 +4,7 @@
 # 官方文档: https://prometheus.io/docs/prometheus/latest/configuration/configuration/
 # ============================================================================

-# global: 全局配置
+# 全局配置
 global:
  scrape_interval: 15s
-  # 指标抓取间隔
-
  evaluation_interval: 15s
-  # 规则评估间隔
@@ -4,124 +4,62 @@
 # 官方文档: https://grafana.com/docs/tempo/latest/configuration/
 # ============================================================================

-# server: HTTP 服务器配置
+# HTTP 服务器配置
 server:
  http_listen_port: 3200
-  # HTTP API 监听端口
-  # 用于查询 Trace 数据和健康检查

-# distributor: 分发器配置
+# 分发器配置
 distributor:
  receivers:
-    # 接收器配置
-
    otlp:
-      # OpenTelemetry Protocol (OTLP) 接收器
-      # OTLP 是 OpenTelemetry 的原生协议，推荐使用
-
      protocols:
        grpc:
          endpoint: "0.0.0.0:4317"
-          # gRPC 协议端点
-
        http:
          endpoint: "0.0.0.0:4318"
-          # HTTP 协议端点

-# storage: 存储配置
+# 存储配置
 storage:
  trace:
    backend: local
-    # 存储后端类型
-    # 生产环境建议使用对象存储
-
    wal:
      path: /tmp/tempo/wal
-      # Write-Ahead Log (WAL) 目录
-      # WAL 用于在数据写入存储前的临时缓存
-      # 确保数据不会因意外重启而丢失
-      # 注意: 生产环境应使用持久化存储路径
-
    local:
      path: /tmp/tempo/blocks
-      # 本地存储的数据块目录
-      # 存储压缩后的 Trace 数据块
-      # 仅在 backend: local 时使用

-# metrics_generator: 指标生成器配置
-# 从 Trace 数据自动生成 Prometheus 指标
-# 这是实现 Trace 到 Metrics 关联的关键功能
+# 指标生成器配置
 metrics_generator:
  registry:
    external_labels:
      source: tempo
-      # 外部标签
-      # 添加到所有生成的指标上

  storage:
    path: /tmp/tempo/generator/wal
-    # 指标生成器的 WAL 目录
-    # 用于临时存储待发送的指标数据
-
    remote_write:
-      # 远程写入配置
-
      - url: http://fba_prometheus:9090/api/v1/write
-        # Prometheus 远程写入 API 地址
-        # 需要 Prometheus 启用 remote-write-receiver 功能
-
        send_exemplars: true
-        # 发送 Exemplar 数据
-        # Exemplar 将指标数据点与 Trace ID 关联
-        # 实现从指标图表点击跳转到对应 Trace

  traces_storage:
    path: /tmp/tempo/generator/traces
-    # Trace 数据临时存储路径
-    # 用于指标生成器处理 Trace 数据

  processor:
-    # 处理器配置
-    # 定义如何从 Trace 生成指标
-
    span_metrics:
-      # Span 指标处理器
-
      dimensions:
-        # 指标维度（标签）
-        # 这些属性将作为 Prometheus 指标的标签
-
        - service.name
        - http.method
        - http.target
        - http.status_code
-
    service_graphs:
-      # 服务图处理器
-      # 生成服务间调用关系的指标
-      # 用于 Grafana 中的服务地图可视化
-
      dimensions:
        - service.name
-
    local_blocks:
-      # 本地块处理器
-      # 处理本地存储的 Trace 数据块
-
      flush_to_storage: true
-      # 是否将处理后的数据刷新到存储

-# overrides: 覆盖配置
+# 覆盖配置
 overrides:
  defaults:
-    # 默认配置
-    # 适用于所有租户（单租户模式下为全局配置）
-
    metrics_generator:
      processors:
-        # 启用的指标处理器列表
-        # 只有在此列表中的处理器才会运行
-
        - span-metrics
        - service-graphs
        - local-blocks
@@ -1,6 +1,6 @@
 [program:fba_server]
 directory=/fba/backend
-command=/usr/local/bin/granian main:app --interface asgi --host 0.0.0.0 --port 8001 --workers 1 --backlog 1024 --workers-kill-timeout 120 --backpressure 2000 --pid-file /var/run/granian.pid --log --log-level debug
+command=/usr/local/bin/granian main:app --interface asgi --host 0.0.0.0 --port 8001 --metrics --metrics-address 0.0.0.0 --metrics-port 8002 --workers 1 --backlog 1024 --workers-kill-timeout 120 --backpressure 2000 --pid-file /var/run/granian.pid --log --log-level debug
 user=root
 autostart=true
 autorestart=true