Add support for opentelemetry metrics (#1245)

* feat(telemetry): initialize OpenTelemetry metrics provider

* feat(metrics): implement and integrate HTTP request duration metrics

* feat(metrics): instrument database query durations

* refactor(database): add OTel name mapping helper

* feat(metrics): instrument database connection pool

* feat(telemetry): update OTel collector to receive OTLP metrics

* feat(telemetry): add metrics panels to Grafana dashboard

* refactor(metrics): clean up database instrumentation with helper functions

* fix(metrics): ensure correct db.system.name for ODBC connections by reusing discovery logic

* fix(metrics): remove global OnceLock and pass database type explicitly to pool callbacks

* refactor(metrics): use OpenTelemetry semantic convention constants

* refactor(metrics): eliminate all OTel convention string literals in favor of constants

* fix(telemetry): use standard YAML list notation in OTel collector config to fix parsing error

* fix(telemetry): fix clippy warnings and use latest OTel attribute names

* fix(telemetry): add Default impl for TelemetryMetrics

* Add app-scoped telemetry metrics for HTTP

* Refactor db query metrics recording

* Use semantic convention attribute names

* Track db pool metrics during lifecycle

* Adjust telemetry docker compose

* fix(metrics): configure explicit histogram boundaries and fix database latency measurement

* Enable metrics views for explicit histograms

* Use observable gauge for pool connection count

* Read pool gauge directly from sqlx pool

* telemetry: export OTEL metrics every second in example stack

* Improve Grafana trace list links and readability

* Restore removed DB connection debug logs

Author: lovasoa

Cargo.lock

@@ -88,8 +88,9 @@ tracing-opentelemetry = "0.32"
 tracing-actix-web = { version = "0.7", default-features = false, features = ["opentelemetry_0_31"] }
 tracing-log = "0.2"
 opentelemetry = "0.31"
-opentelemetry_sdk = { version = "0.31", features = ["rt-tokio-current-thread"] }
-opentelemetry-otlp = { version = "0.31", features = ["http-proto", "grpc-tonic"] }
+opentelemetry_sdk = { version = "0.31", features = ["metrics", "rt-tokio-current-thread", "spec_unstable_metrics_views"] }
+opentelemetry-otlp = { version = "0.31", features = ["http-proto", "grpc-tonic", "metrics"] }
+opentelemetry-semantic-conventions = { version = "0.31", features = ["semconv_experimental"] }
 
 
 [features]

Cargo.toml

@@ -23,6 +23,7 @@ services:
     environment:
       - DATABASE_URL=postgres://sqlpage:sqlpage@postgres:5432/sqlpage
       - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
+      - OTEL_METRIC_EXPORT_INTERVAL=1000
       - OTEL_SERVICE_NAME=sqlpage
     volumes:
       - ./website:/var/www
@@ -113,8 +114,10 @@ services:
       - "1514:1514/udp"
       - "1516:1516/udp"
     depends_on:
-      - tempo
-      - postgres
+      tempo:
+        condition: service_started
+      postgres:
+        condition: service_started
       loki:
         condition: service_healthy
 

examples/telemetry/docker-compose.yml

@@ -39,7 +39,7 @@
           "showLineNumbers": false,
           "showMiniMap": false
         },
-        "content": "<div style=\"padding: 4px 2px 0; font-size: 15px; line-height: 1.55;\"><h1 style=\"margin: 0 0 10px; font-size: 28px;\">Recent SQLPage traces, logs, and PostgreSQL metrics</h1><p style=\"margin: 0 0 8px;\">Open <a href=\"http://localhost\" target=\"_blank\" rel=\"noopener noreferrer\">http://localhost</a> and interact with the app. New requests will appear here automatically.</p><p style=\"margin: 0; color: #666;\">The trace table shows recent requests. Click any trace ID to open the full span waterfall in Grafana. PostgreSQL slow-query explain plans appear in the PostgreSQL Logs panel and link back to the same trace via the extracted trace ID. The metrics panels come from the OpenTelemetry PostgreSQL receiver via Prometheus.</p></div>",
+        "content": "<div style=\"padding: 4px 2px 0; font-size: 15px; line-height: 1.55;\"><h1 style=\"margin: 0 0 10px; font-size: 28px;\">SQLPage Observability</h1><p style=\"margin: 0 0 8px;\">Open <a href=\"http://localhost\" target=\"_blank\" rel=\"noopener noreferrer\">http://localhost</a> and interact with the app. New requests will appear here automatically.</p><p style=\"margin: 0; color: #666;\">This dashboard shows traces, logs, and application metrics exported by SQLPage. Trace waterfalls link to PostgreSQL logs via trace IDs. Metrics include HTTP durations, DB query latencies, and connection pool states.</p></div>",
         "mode": "html"
       },
       "pluginVersion": "12.4.0",
@@ -54,7 +54,39 @@
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "thresholds"
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 2,
+            "pointSize": 4,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
           },
           "mappings": [],
           "thresholds": {
@@ -63,38 +95,31 @@
               {
                 "color": "green",
                 "value": null
-              },
-              {
-                "color": "orange",
-                "value": 10
               }
             ]
           },
-          "unit": "none"
+          "unit": "s"
         },
         "overrides": []
       },
       "gridPos": {
-        "h": 4,
+        "h": 8,
         "w": 12,
         "x": 0,
         "y": 4
       },
-      "id": 4,
+      "id": 10,
       "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "percentChangeColorMode": "standard",
-        "reduceOptions": {
-          "calcs": ["lastNotNull"],
-          "fields": "",
-          "values": false
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
         },
-        "showPercentChange": false,
-        "textMode": "auto",
-        "wideLayout": true
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
       },
       "pluginVersion": "12.4.0",
       "targets": [
@@ -103,12 +128,22 @@
             "type": "prometheus",
             "uid": "prometheus"
           },
-          "expr": "sum(postgresql_backends)",
+          "expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket[5m])) by (le, http_route))",
+          "legendFormat": "HTTP P95 {{http_route}}",
           "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum(rate(db_client_operation_duration_seconds_bucket[5m])) by (le, db_operation_name))",
+          "legendFormat": "DB P95 {{db_operation_name}}",
+          "refId": "B"
         }
       ],
-      "title": "PostgreSQL Backends",
-      "type": "stat"
+      "title": "Request & Query Latency (P95)",
+      "type": "timeseries"
     },
     {
       "datasource": {
@@ -162,17 +197,17 @@
               }
             ]
           },
-          "unit": "bytes"
+          "unit": "none"
         },
         "overrides": []
       },
       "gridPos": {
-        "h": 4,
+        "h": 8,
         "w": 12,
         "x": 12,
         "y": 4
       },
-      "id": 5,
+      "id": 11,
       "options": {
         "legend": {
           "calcs": [],
@@ -192,12 +227,12 @@
             "type": "prometheus",
             "uid": "prometheus"
           },
-          "expr": "sum(postgresql_db_size) by (postgresql_database_name)",
-          "legendFormat": "{{postgresql_database_name}}",
+          "expr": "sum(db_client_connection_count) by (db_client_connection_state)",
+          "legendFormat": "{{db_client_connection_state}}",
           "refId": "A"
         }
       ],
-      "title": "PostgreSQL Database Size",
+      "title": "SQLPage DB Connection Pool",
       "type": "timeseries"
     },
     {
@@ -223,8 +258,8 @@
             },
             "properties": [
               {
-                "id": "custom.width",
-                "value": 300
+                "id": "custom.hidden",
+                "value": true
               }
             ]
           },
@@ -248,7 +283,35 @@
             "properties": [
               {
                 "id": "custom.width",
-                "value": 140
+                "value": 120
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "traceName"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 520
+              },
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "type": "data-links"
+                }
+              },
+              {
+                "id": "links",
+                "value": [
+                  {
+                    "targetBlank": false,
+                    "title": "${__value.text}",
+                    "url": "/a/grafana-exploretraces-app/explore?traceId=${__data.fields.traceID}"
+                  }
+                ]
               }
             ]
           },
@@ -279,10 +342,10 @@
         ]
       },
       "gridPos": {
-        "h": 12,
+        "h": 8,
         "w": 24,
         "x": 0,
-        "y": 8
+        "y": 12
       },
       "id": 2,
       "options": {
@@ -335,7 +398,7 @@
             "renameByName": {
               "startTime": "Start time",
               "traceDuration": "Duration",
-              "traceID": "Trace ID",
+              "traceID": "Trace",
               "traceName": "Route",
               "traceService": "Service"
             }
@@ -344,6 +407,36 @@
       ],
       "type": "table"
     },
+    {
+      "datasource": {
+        "type": "tempo",
+        "uid": "tempo"
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 30
+      },
+      "id": 12,
+      "pluginVersion": "12.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "tempo",
+            "uid": "tempo"
+          },
+          "limit": 20,
+          "query": "$traceId",
+          "queryType": "traceql",
+          "refId": "A",
+          "tableType": "traces"
+        }
+      ],
+      "timeFrom": "1h",
+      "title": "Selected Trace",
+      "type": "traces"
+    },
     {
       "datasource": {
         "type": "loki",
@@ -428,9 +521,30 @@
   "refresh": "5s",
   "schemaVersion": 41,
   "style": "dark",
-  "tags": ["sqlpage", "tracing", "logs"],
+  "tags": ["sqlpage", "tracing", "logs", "metrics"],
   "templating": {
-    "list": []
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "text": "",
+          "value": ""
+        },
+        "hide": 2,
+        "label": "Trace ID",
+        "name": "traceId",
+        "options": [
+          {
+            "selected": true,
+            "text": "",
+            "value": ""
+          }
+        ],
+        "query": "",
+        "skipUrlSync": false,
+        "type": "textbox"
+      }
+    ]
   },
   "time": {
     "from": "now-1h",
@@ -440,5 +554,5 @@
   "timezone": "browser",
   "title": "SQLPage Observability Home",
   "uid": "sqlpage-tracing-home",
-  "version": 5
+  "version": 6
 }