feat: local observability playground

README.md

@@ -126,27 +126,7 @@ cargo test --package op-rbuilder --lib
 
 ## Local Devnet
 
-1. Install [flashbots/builder-playground](https://github.com/flashbots/builder-playground).
-
-2. `builder-playground start playground.yaml` (use `--skip-setup` if you need to skip `cargo build` every time).
-
-3. Run `contender`:
-
-```bash
-cargo run -- spam --tps 10 -r http://localhost:2222 --optimism --min-balance 0.14
-```
-
-And you should start to see blocks being built and landed on-chain with `contender` transactions.
-
-Alternatively, you can send a single test transaction like:
-
-```bash
-builder-playground test http://localhost:2222 --timeout 30s --retries 10
-```
-
-op-rbuilder will automatically try to detect all settings and ports from the currently running playground thanks to the `--builder.playground` flag.
-
-Make sure to check out `playground.yaml` if you need to inspect or modify the configuration for your local test environment!
+Local development environment setup and configuration files can be found in [`dev/README.md`](./dev/README.md).
 
 ## Running GitHub actions locally
 

crates/op-rbuilder/src/builder/payload.rs

@@ -474,18 +474,9 @@ where
             cancel: block_cancel,
         } = args;
 
-        // We log only every 100th block to reduce usage
-        let span = if cfg!(feature = "telemetry")
-            && config
-                .parent_header
-                .number
-                .is_multiple_of(self.config.sampling_ratio)
-        {
-            span!(Level::INFO, "build_payload")
-        } else {
-            tracing::Span::none()
-        };
-        let _entered = span.enter();
+        // The build_payload span is created and instrumented in try_build() using
+        // tracing::Instrument, which safely manages it across async .await points.
+        let span = tracing::Span::current();
         span.record(
             "payload_id",
             config.attributes.payload_attributes.id.to_string(),
@@ -747,8 +738,6 @@ where
                     "build_flashblock",
                 )
             };
-            let _entered = fb_span.enter();
-
             let FlashblockBuildOutput {
                 ctx: returned_ctx,
                 build_result,
@@ -757,8 +746,8 @@ where
                 committed_txs: new_committed,
                 info: new_info,
                 fb_state: returned_fb_state,
-            } = self
-                .run_blocking_task({
+            } = tracing::Instrument::instrument(
+                self.run_blocking_task({
                     let builder = self.clone();
                     let ctx = ctx;
                     let block_cancel = block_cancel.clone();
@@ -804,8 +793,10 @@ where
                             fb_state,
                         })
                     }
-                })
-                .await?;
+                }),
+                fb_span,
+            )
+            .await?;
 
             ctx = returned_ctx;
             fb_state = returned_fb_state;
@@ -1133,7 +1124,18 @@ where
         args: BuildArguments<Self::Attributes, Self::BuiltPayload>,
         best_payload_tx: watch::Sender<Option<Self::BuiltPayload>>,
     ) -> Result<(), PayloadBuilderError> {
-        self.build_payload(args, best_payload_tx).await
+        let span = if cfg!(feature = "telemetry")
+            && args
+                .config
+                .parent_header
+                .number
+                .is_multiple_of(self.config.sampling_ratio)
+        {
+            span!(Level::INFO, "build_payload")
+        } else {
+            tracing::Span::none()
+        };
+        tracing::Instrument::instrument(self.build_payload(args, best_payload_tx), span).await
     }
 }
 

crates/op-rbuilder/src/launcher.rs

@@ -50,6 +50,9 @@ pub fn launch() -> Result<()> {
         use crate::primitives::telemetry::setup_telemetry_layer;
         let telemetry_layer = setup_telemetry_layer(&telemetry_args)?;
         cli_app.access_tracing_layers()?.add_layer(telemetry_layer);
+
+        // macos fix
+        otel_shutdown_hook();
     }
 
     cli_app.run(BuilderLauncher)?;
@@ -167,3 +170,26 @@ impl Launcher<OpChainSpecParser, OpRbuilderArgs> for BuilderLauncher {
         Ok(())
     }
 }
+
+/// Panic hook for known macOS TLS destruction ordering crash OpenTelemetry
+#[cfg(feature = "telemetry")]
+fn otel_shutdown_hook() {
+    let default_hook = std::panic::take_hook();
+    std::panic::set_hook(Box::new(move |info| {
+        let is_tls_panic = info
+            .payload()
+            .downcast_ref::<String>()
+            .map(|s| s.contains("Thread Local Storage value during or after destruction"))
+            .or_else(|| {
+                info.payload()
+                    .downcast_ref::<&str>()
+                    .map(|s| s.contains("Thread Local Storage value during or after destruction"))
+            })
+            .unwrap_or(false);
+
+        if is_tls_panic {
+            std::process::exit(0);
+        }
+        default_hook(info);
+    }));
+}

crates/op-rbuilder/src/primitives/telemetry.rs

@@ -19,9 +19,13 @@ pub fn setup_telemetry_layer(
     }
 
     // Create OTLP layer with custom configuration
+    let mut endpoint =
+        Url::parse(args.otlp_endpoint.as_ref().unwrap()).expect("Invalid OTLP endpoint");
+    reth_tracing_otlp::OtlpProtocol::Http.validate_endpoint(&mut endpoint)?;
+
     let config = reth_tracing_otlp::OtlpConfig::new(
         "op-rbuilder",
-        Url::parse(args.otlp_endpoint.as_ref().unwrap()).expect("Invalid OTLP endpoint"),
+        endpoint,
         reth_tracing_otlp::OtlpProtocol::Http,
         None,
     )?;

dev/README.md

@@ -0,0 +1,48 @@
+# Local Dev Environment
+
+This directory contains local development environment configuration for `op-rbuilder`.
+
+## Prerequisites
+
+1. [flashbots/builder-playground](https://github.com/flashbots/builder-playground).
+1. [flashbots/contender](https://github.com/flashbots/contender).
+
+## Start Local Devnet
+
+From the repository root:
+
+```bash
+builder-playground start dev/playground.yaml
+```
+
+Use `--skip-setup` to skip `cargo build` on repeated runs:
+
+```bash
+builder-playground start dev/playground.yaml --skip-setup
+```
+
+Services:
+- Grafana:    http://localhost:3000
+- Jaeger:     http://localhost:16686
+- Prometheus: http://localhost:9090
+- RPC:        http://localhost:2222
+
+Logs can be found at `$HOME/.local/state/builder-playground/sessions/latest/logs/`
+
+## Generate Local Traffic
+
+Run `contender`:
+
+```bash
+contender spam --tps 10 -r http://localhost:2222 --optimism --min-balance 0.14
+```
+
+You should start to see blocks being built and landed on-chain with `contender` transactions.
+
+Alternatively, send a single test transaction:
+
+```bash
+builder-playground test http://localhost:2222 --timeout 30s --retries 10
+```
+
+`op-rbuilder` automatically detects settings and ports from the running playground via `--builder.playground`.

dev/grafana/datasources.yaml

@@ -0,0 +1,17 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
+
+  - name: Jaeger
+    type: jaeger
+    uid: jaeger
+    access: proxy
+    url: http://jaeger:16686
+    editable: false

playground.yaml → dev/playground.yaml

@@ -4,13 +4,13 @@ base_args:
   - op-rbuilder
 
 setup:
-  - make op-rbuilder
+  - cd .. && make op-rbuilder FEATURES=telemetry
 
 recipe:
   op-rbuilder:
     services:
       op-rbuilder:
-        host_path: ./target/debug/op-rbuilder
+        host_path: ../target/debug/op-rbuilder
         args:
           - node
           - --builder.playground # enabling op-rbuilder to detect playground environment and service configurations
@@ -41,6 +41,12 @@ recipe:
           - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
           - --bootnodes
           - '{{Service "bootnode" "rpc" "enode" "79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8"}}'
+          - --log.stdout.filter
+          - op_rbuilder=debug
+          - --telemetry.otlp-endpoint
+          - http://localhost:4318
+          - --telemetry.sampling-ratio
+          - '1'
         ports:
           authrpc: 4444
           http: 2222
@@ -52,3 +58,37 @@ recipe:
         volumes:
           /data_op_rbuilder:
             name: data # $HOME/.local/state/builder-playground/devnet/volume-op-rbuilder-data
+
+  observability:
+    services:
+      jaeger:
+        image: jaegertracing/all-in-one
+        tag: latest
+        ports:
+          otlp-http: 4318
+          ui: 16686
+        env:
+          COLLECTOR_OTLP_ENABLED: "true"
+
+      prometheus:
+        image: prom/prometheus
+        tag: latest
+        args:
+          - --config.file=/etc/prometheus/prometheus.yml
+          - --web.enable-lifecycle
+          - --storage.tsdb.retention.time=1h
+        ports:
+          http: 9090
+        files:
+          /etc/prometheus/prometheus.yml: prometheus.yaml
+
+      grafana:
+        image: grafana/grafana
+        tag: latest
+        ports:
+          http: 3000
+        env:
+          GF_AUTH_ANONYMOUS_ENABLED: "true"
+          GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
+        files:
+          /etc/grafana/provisioning/datasources/datasources.yaml: grafana/datasources.yaml

dev/prometheus.yaml

@@ -0,0 +1,24 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+
+scrape_configs:
+  - job_name: op-rbuilder
+    metrics_path: /metrics
+    static_configs:
+      - targets: ['host.docker.internal:9011']
+
+  - job_name: el
+    metrics_path: /metrics
+    static_configs:
+      - targets: ['el:9090']
+
+  - job_name: op-geth
+    metrics_path: /debug/metrics/prometheus
+    static_configs:
+      - targets: ['op-geth:6061']
+
+  - job_name: op-node
+    metrics_path: /metrics
+    static_configs:
+      - targets: ['op-node:7300']