diff --git a/README.md b/README.md index 27e277b1..92cd5fcc 100644 --- a/README.md +++ b/README.md @@ -126,27 +126,7 @@ cargo test --package op-rbuilder --lib ## Local Devnet -1. Install [flashbots/builder-playground](https://github.com/flashbots/builder-playground). - -2. `builder-playground start playground.yaml` (use `--skip-setup` if you need to skip `cargo build` every time). - -3. Run `contender`: - -```bash -cargo run -- spam --tps 10 -r http://localhost:2222 --optimism --min-balance 0.14 -``` - -And you should start to see blocks being built and landed on-chain with `contender` transactions. - -Alternatively, you can send a single test transaction like: - -```bash -builder-playground test http://localhost:2222 --timeout 30s --retries 10 -``` - -op-rbuilder will automatically try to detect all settings and ports from the currently running playground thanks to the `--builder.playground` flag. - -Make sure to check out `playground.yaml` if you need to inspect or modify the configuration for your local test environment! +Local development environment setup and configuration files can be found in [`dev/README.md`](./dev/README.md). ## Running GitHub actions locally diff --git a/crates/op-rbuilder/src/builder/payload.rs b/crates/op-rbuilder/src/builder/payload.rs index 64d469c1..0968c8dc 100644 --- a/crates/op-rbuilder/src/builder/payload.rs +++ b/crates/op-rbuilder/src/builder/payload.rs @@ -474,18 +474,9 @@ where cancel: block_cancel, } = args; - // We log only every 100th block to reduce usage - let span = if cfg!(feature = "telemetry") - && config - .parent_header - .number - .is_multiple_of(self.config.sampling_ratio) - { - span!(Level::INFO, "build_payload") - } else { - tracing::Span::none() - }; - let _entered = span.enter(); + // The build_payload span is created and instrumented in try_build() using + // tracing::Instrument, which safely manages it across async .await points. + let span = tracing::Span::current(); span.record( "payload_id", config.attributes.payload_attributes.id.to_string(), @@ -747,8 +738,6 @@ where "build_flashblock", ) }; - let _entered = fb_span.enter(); - let FlashblockBuildOutput { ctx: returned_ctx, build_result, @@ -757,8 +746,8 @@ where committed_txs: new_committed, info: new_info, fb_state: returned_fb_state, - } = self - .run_blocking_task({ + } = tracing::Instrument::instrument( + self.run_blocking_task({ let builder = self.clone(); let ctx = ctx; let block_cancel = block_cancel.clone(); @@ -804,8 +793,10 @@ where fb_state, }) } - }) - .await?; + }), + fb_span, + ) + .await?; ctx = returned_ctx; fb_state = returned_fb_state; @@ -1133,7 +1124,18 @@ where args: BuildArguments, best_payload_tx: watch::Sender>, ) -> Result<(), PayloadBuilderError> { - self.build_payload(args, best_payload_tx).await + let span = if cfg!(feature = "telemetry") + && args + .config + .parent_header + .number + .is_multiple_of(self.config.sampling_ratio) + { + span!(Level::INFO, "build_payload") + } else { + tracing::Span::none() + }; + tracing::Instrument::instrument(self.build_payload(args, best_payload_tx), span).await } } diff --git a/crates/op-rbuilder/src/launcher.rs b/crates/op-rbuilder/src/launcher.rs index a165426f..f52b856e 100644 --- a/crates/op-rbuilder/src/launcher.rs +++ b/crates/op-rbuilder/src/launcher.rs @@ -50,6 +50,9 @@ pub fn launch() -> Result<()> { use crate::primitives::telemetry::setup_telemetry_layer; let telemetry_layer = setup_telemetry_layer(&telemetry_args)?; cli_app.access_tracing_layers()?.add_layer(telemetry_layer); + + // macos fix + otel_shutdown_hook(); } cli_app.run(BuilderLauncher)?; @@ -167,3 +170,26 @@ impl Launcher for BuilderLauncher { Ok(()) } } + +/// Panic hook for known macOS TLS destruction ordering crash OpenTelemetry +#[cfg(feature = "telemetry")] +fn otel_shutdown_hook() { + let default_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + let is_tls_panic = info + .payload() + .downcast_ref::() + .map(|s| s.contains("Thread Local Storage value during or after destruction")) + .or_else(|| { + info.payload() + .downcast_ref::<&str>() + .map(|s| s.contains("Thread Local Storage value during or after destruction")) + }) + .unwrap_or(false); + + if is_tls_panic { + std::process::exit(0); + } + default_hook(info); + })); +} diff --git a/crates/op-rbuilder/src/primitives/telemetry.rs b/crates/op-rbuilder/src/primitives/telemetry.rs index 020f52e4..a5058876 100644 --- a/crates/op-rbuilder/src/primitives/telemetry.rs +++ b/crates/op-rbuilder/src/primitives/telemetry.rs @@ -19,9 +19,13 @@ pub fn setup_telemetry_layer( } // Create OTLP layer with custom configuration + let mut endpoint = + Url::parse(args.otlp_endpoint.as_ref().unwrap()).expect("Invalid OTLP endpoint"); + reth_tracing_otlp::OtlpProtocol::Http.validate_endpoint(&mut endpoint)?; + let config = reth_tracing_otlp::OtlpConfig::new( "op-rbuilder", - Url::parse(args.otlp_endpoint.as_ref().unwrap()).expect("Invalid OTLP endpoint"), + endpoint, reth_tracing_otlp::OtlpProtocol::Http, None, )?; diff --git a/dev/README.md b/dev/README.md new file mode 100644 index 00000000..b264bae4 --- /dev/null +++ b/dev/README.md @@ -0,0 +1,48 @@ +# Local Dev Environment + +This directory contains local development environment configuration for `op-rbuilder`. + +## Prerequisites + +1. [flashbots/builder-playground](https://github.com/flashbots/builder-playground). +1. [flashbots/contender](https://github.com/flashbots/contender). + +## Start Local Devnet + +From the repository root: + +```bash +builder-playground start dev/playground.yaml +``` + +Use `--skip-setup` to skip `cargo build` on repeated runs: + +```bash +builder-playground start dev/playground.yaml --skip-setup +``` + +Services: +- Grafana: http://localhost:3000 +- Jaeger: http://localhost:16686 +- Prometheus: http://localhost:9090 +- RPC: http://localhost:2222 + +Logs can be found at `$HOME/.local/state/builder-playground/sessions/latest/logs/` + +## Generate Local Traffic + +Run `contender`: + +```bash +contender spam --tps 10 -r http://localhost:2222 --optimism --min-balance 0.14 +``` + +You should start to see blocks being built and landed on-chain with `contender` transactions. + +Alternatively, send a single test transaction: + +```bash +builder-playground test http://localhost:2222 --timeout 30s --retries 10 +``` + +`op-rbuilder` automatically detects settings and ports from the running playground via `--builder.playground`. diff --git a/dev/grafana/datasources.yaml b/dev/grafana/datasources.yaml new file mode 100644 index 00000000..6e0ba840 --- /dev/null +++ b/dev/grafana/datasources.yaml @@ -0,0 +1,17 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + + - name: Jaeger + type: jaeger + uid: jaeger + access: proxy + url: http://jaeger:16686 + editable: false diff --git a/playground.yaml b/dev/playground.yaml similarity index 58% rename from playground.yaml rename to dev/playground.yaml index 60bee48c..b08adf00 100755 --- a/playground.yaml +++ b/dev/playground.yaml @@ -4,13 +4,13 @@ base_args: - op-rbuilder setup: - - make op-rbuilder + - cd .. && make op-rbuilder FEATURES=telemetry recipe: op-rbuilder: services: op-rbuilder: - host_path: ./target/debug/op-rbuilder + host_path: ../target/debug/op-rbuilder args: - node - --builder.playground # enabling op-rbuilder to detect playground environment and service configurations @@ -41,6 +41,12 @@ recipe: - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 - --bootnodes - '{{Service "bootnode" "rpc" "enode" "79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8"}}' + - --log.stdout.filter + - op_rbuilder=debug + - --telemetry.otlp-endpoint + - http://localhost:4318 + - --telemetry.sampling-ratio + - '1' ports: authrpc: 4444 http: 2222 @@ -52,3 +58,37 @@ recipe: volumes: /data_op_rbuilder: name: data # $HOME/.local/state/builder-playground/devnet/volume-op-rbuilder-data + + observability: + services: + jaeger: + image: jaegertracing/all-in-one + tag: latest + ports: + otlp-http: 4318 + ui: 16686 + env: + COLLECTOR_OTLP_ENABLED: "true" + + prometheus: + image: prom/prometheus + tag: latest + args: + - --config.file=/etc/prometheus/prometheus.yml + - --web.enable-lifecycle + - --storage.tsdb.retention.time=1h + ports: + http: 9090 + files: + /etc/prometheus/prometheus.yml: prometheus.yaml + + grafana: + image: grafana/grafana + tag: latest + ports: + http: 3000 + env: + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Admin + files: + /etc/grafana/provisioning/datasources/datasources.yaml: grafana/datasources.yaml diff --git a/dev/prometheus.yaml b/dev/prometheus.yaml new file mode 100644 index 00000000..32b4c384 --- /dev/null +++ b/dev/prometheus.yaml @@ -0,0 +1,24 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + - job_name: op-rbuilder + metrics_path: /metrics + static_configs: + - targets: ['host.docker.internal:9011'] + + - job_name: el + metrics_path: /metrics + static_configs: + - targets: ['el:9090'] + + - job_name: op-geth + metrics_path: /debug/metrics/prometheus + static_configs: + - targets: ['op-geth:6061'] + + - job_name: op-node + metrics_path: /metrics + static_configs: + - targets: ['op-node:7300']