Configured Grafana Tempo for traces in addition to Jaeger

Author: Mikalai Alimenkou

compose.yaml

@@ -34,6 +34,17 @@ services:
       retries: 5
       start_period: 10s
 
+  tempo:
+    image: grafana/tempo:2.9.0
+    container_name: library-tempo
+    command: [ "-config.file=/etc/tempo/tempo.yaml" ]
+    volumes:
+      - ./tempo/tempo-config.yaml:/etc/tempo/tempo.yaml:ro
+    ports:
+      - "3200:3200"   # Tempo HTTP API
+      - "4311:4317"   # OTLP gRPC
+      - "4312:4318"   # OTLP HTTP
+
   otel-collector:
     image: otel/opentelemetry-collector-contrib:0.137.0
     container_name: library-otel-collector
@@ -47,6 +58,8 @@ services:
     depends_on:
       loki:
         condition: service_healthy
+      tempo:
+        condition: service_started
 
   prometheus:
     image: prom/prometheus:v3.7.1
@@ -57,6 +70,8 @@ services:
       - '--web.console.libraries=/etc/prometheus/console_libraries'
       - '--web.console.templates=/etc/prometheus/consoles'
       - '--enable-feature=exemplar-storage'
+      - '--web.enable-remote-write-receiver'
+      - '--enable-feature=native-histograms'
     volumes:
       - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
     ports:
@@ -78,6 +93,7 @@ services:
     depends_on:
       - prometheus
       - loki
+      - tempo
 
   library-app:
     profiles: ["perftest"]
@@ -104,11 +120,9 @@ services:
       interval: 10s
       timeout: 5s
       retries: 3
-      start_period: 30s
+      start_period: 10s
     depends_on:
       db:
         condition: service_started
-      jaeger:
-        condition: service_started
-      loki:
-        condition: service_healthy
+      otel-collector:
+        condition: service_started

grafana/datasources/datasources.yml

@@ -3,24 +3,61 @@ apiVersion: 1
 datasources:
   - name: Prometheus
     type: prometheus
+    uid: prometheus
     access: proxy
     url: http://prometheus:9090
     isDefault: true
     editable: true
     jsonData:
       timeInterval: 5s
       queryTimeout: 60s
+      exemplarTraceIdDestinations:
+        - name: trace_id
+          datasourceUid: tempo
+          datasourceName: Tempo
+          urlDisplayLabel: "View trace in Tempo"
 
   - name: Loki
     type: loki
+    uid: loki
     access: proxy
     url: http://loki:3100
     isDefault: false
     editable: true
     jsonData:
-      maxLines: 1000
       derivedFields:
         - datasourceUid: tempo
-          matcherRegex: "trace_id=(\\w+)"
+          datasourceName: Tempo
+          matcherType: label
+          matcherRegex: trace_id
           name: TraceID
           url: "$${__value.raw}"
+          urlDisplayLabel: "View trace in Tempo"
+
+  - name: Tempo
+    type: tempo
+    uid: tempo
+    access: proxy
+    url: http://tempo:3200
+    isDefault: false
+    editable: true
+    jsonData:
+      httpMethod: GET
+      serviceMap:
+        datasourceUid: prometheus
+      spanMetrics:
+        datasourceUid: prometheus
+      streamingEnabled:
+        search: true
+      tracesToLogsV2:
+        datasourceUid: loki
+        datasourceName: Loki
+        filterBySpanID: false
+        filterByTraceID: true
+        spanStartTimeShift: "-1h"
+        spanEndTimeShift: "1h"
+        tags:
+          - key: service.name
+            value: service_name
+        customQuery: true
+        query: '{$${__tags}} | trace_id="$${__span.traceId}"'

otel/otel-collector-config.yaml

@@ -113,6 +113,11 @@ exporters:
     tls:
       insecure: true
 
+  otlphttp/tempo:
+    endpoint: http://tempo:4318
+    tls:
+      insecure: true
+
 service:
   extensions: [file_storage]
   pipelines:
@@ -123,4 +128,4 @@ service:
     traces:
       receivers: [otlp]
       processors: [batch]
-      exporters: [otlphttp]
+      exporters: [otlphttp, otlphttp/tempo, debug]

tempo/tempo-config.yaml

@@ -0,0 +1,73 @@
+stream_over_http_enabled: true
+server:
+  http_listen_port: 3200
+  log_level: info
+
+query_frontend:
+  mcp_server:
+    enabled: true
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+    metadata_slo:
+      duration_slo: 5s
+      throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 100ms
+  metrics:
+    max_duration: 200h
+    query_backend_after: 5m
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+
+distributor:
+  usage:
+    cost_attribution:
+      enabled: true
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:4317
+        http:
+          endpoint: 0.0.0.0:4318
+
+ingester:
+  lifecycler:
+    min_ready_duration: 0s
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    block_retention: 720h
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+  traces_storage:
+    path: /var/tempo/generator/traces
+  processor:
+    local_blocks:
+      filter_server_spans: false
+      flush_to_storage: true
+
+storage:
+  trace:
+    backend: local
+    wal:
+      path: /var/tempo/wal
+    local:
+      path: /var/tempo/blocks
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics, local-blocks]
+      generate_native_histograms: both