diff --git a/AGENTS.md b/AGENTS.md index 7633872..413859c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -90,17 +90,15 @@ Use conditional compilation for feature-specific code: #[cfg_attr(feature = "cli", derive(clap::Args))] pub struct MyArgs { ... } -#[cfg_attr(feature = "display", derive(tabled::Tabled))] +// Display (tabled) is always available with lib feature +#[derive(tabled::Tabled)] pub struct MyResult { ... } ``` -Feature tiers (each includes the previous): -- `database`: SQLite operations only -- `lens-core`: Standalone lenses (TimeLens) -- `lens-bgpkit`: BGP-related lenses -- `lens-full`: All lenses including InspectLens -- `display`: Table formatting with tabled -- `cli`: Full CLI binary with server support +Feature flags (3 simple options): +- `lib`: Complete library (database + all lenses + display) +- `server`: WebSocket server (implies lib) +- `cli`: Full CLI binary (implies lib and server) ## Project Structure @@ -160,8 +158,11 @@ mod tests { - Keep language factual and professional - Avoid words like "comprehensive", "extensive", "amazing", "powerful", "robust" - Use objective language: "Added X", "Fixed Y", "Updated Z" -- Update CHANGELOG.md for fixes/features in "Unreleased changes" section -- When changing `lib.rs` docs, run: `cargo readme > README.md` +- **Update CHANGELOG.md for every commit** - Add entries to "Unreleased changes" section for: + - Breaking changes + - New features + - Bug fixes + - Code improvements - When pushing commits, list all commits first using `git log --oneline origin/[branch]..HEAD` and ask for confirmation ## Common Patterns diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 310de75..00aa4f1 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -141,7 +141,7 @@ src/ ├── monocle.rs # CLI entry point └── commands/ # Command handlers (thin wrappers around lenses) ├── as2rel.rs - ├── config.rs # Config display + db-refresh, db-backup, db-sources + ├── config.rs # Config display + update, backup, sources ├── country.rs ├── inspect.rs # Unified inspect command (replaces whois, pfx2as) ├── ip.rs @@ -319,109 +319,72 @@ The CLI should not duplicate core logic. It should: ## Feature Flags -Monocle supports conditional compilation via Cargo features, organized in tiers: +Monocle supports conditional compilation via Cargo features with a simplified three-tier structure: ### Feature Hierarchy ``` cli (default) - └── lens-full - └── lens-bgpkit - ├── lens-core - │ └── database - └── display + ├── server + │ └── lib + └── lib ``` +**Quick Guide:** +- **Need the CLI binary?** Use `cli` (includes everything) +- **Need WebSocket server without CLI?** Use `server` (includes lib) +- **Need only library/data access?** Use `lib` (database + all lenses + display) + ### Feature Descriptions | Feature | Description | Key Dependencies | |---------|-------------|------------------| -| `database` | SQLite database operations, data loading from URLs | `rusqlite`, `oneio`, `ipnet` | -| `lens-core` | Standalone lenses (TimeLens, OutputFormat) | `chrono-humanize`, `dateparser`, `humantime` | -| `lens-bgpkit` | BGP-related lenses (Parse, Search, RPKI, Country, etc.) | `bgpkit-parser`, `bgpkit-broker`, `bgpkit-commons`, `rayon` | -| `lens-full` | All lenses including InspectLens | (same as lens-bgpkit) | -| `display` | Table formatting with tabled | `tabled`, `json_to_table` | -| `cli` | Full CLI binary with WebSocket server | `clap`, `axum`, `tokio`, `indicatif` | +| `lib` | Complete library: database + all lenses + display | `rusqlite`, `bgpkit-parser`, `bgpkit-broker`, `tabled`, etc. | +| `server` | WebSocket server (implies `lib`) | `axum`, `tokio`, `serde_json` | +| `cli` | Full CLI binary with progress bars (implies `lib` and `server`) | `clap`, `indicatif` | ### Use Case Scenarios -#### Scenario 1: Minimal Library - Database Only -**Features**: `database` +#### Scenario 1: Library Only +**Features**: `lib` -Use when you only need to: -- Access the MonocleDatabase directly -- Query ASInfo, AS2Rel, RPKI, or Pfx2as repositories -- Load data from URLs into the database +Use when building applications that need: +- Database operations (SQLite, data loading) +- All lenses (TimeLens, ParseLens, SearchLens, RPKI, Country, InspectLens, etc.) +- Table formatting with tabled ```toml -monocle = { version = "1.0", default-features = false, features = ["database"] } +monocle = { version = "1.0", default-features = false, features = ["lib"] } ``` ```rust use monocle::database::MonocleDatabase; -let db = MonocleDatabase::open_in_dir("~/.monocle")?; -let asinfo = db.asinfo().get_by_asn(13335)?; -``` - -#### Scenario 2: Time Utilities Only -**Features**: `lens-core` - -Use when you need: -- Time parsing from human-readable strings -- Duration calculations -- Output formatting utilities - -```toml -monocle = { version = "1.0", default-features = false, features = ["lens-core"] } -``` +use monocle::lens::inspect::{InspectLens, InspectQueryOptions}; -```rust -use monocle::lens::time::{TimeLens, TimeParseArgs}; -let lens = TimeLens::new(); -let result = lens.parse(&TimeParseArgs::new("2 hours ago"))?; +let db = MonocleDatabase::open_in_dir("~/.monocle")?; +let lens = InspectLens::new(&db); +let result = lens.query("AS13335", &InspectQueryOptions::default())?; ``` -#### Scenario 3: BGP Operations Without CLI -**Features**: `lens-bgpkit` +#### Scenario 2: Library with WebSocket Server +**Features**: `server` Use when building applications that need: -- MRT file parsing -- BGP message search via bgpkit-broker -- RPKI validation -- Country lookups -- AS relationship queries +- Everything in `lib` +- WebSocket server for remote API access ```toml -monocle = { version = "1.0", default-features = false, features = ["lens-bgpkit"] } +monocle = { version = "1.0", default-features = false, features = ["server"] } ``` ```rust -use monocle::lens::parse::{ParseLens, ParseFilters}; -let lens = ParseLens::new(); -let filters = ParseFilters { - origin_asn: vec!["13335".to_string()], - ..Default::default() -}; -let elems = lens.parse(&filters, "path/to/file.mrt")?; -``` +use monocle::server::start_server; -#### Scenario 4: Full Library Without CLI -**Features**: `lens-full` (or `lens-full,display` for table formatting) - -Use when building applications that need all lens functionality including the unified InspectLens: - -```toml -monocle = { version = "1.0", default-features = false, features = ["lens-full", "display"] } -``` - -```rust -use monocle::lens::inspect::{InspectLens, InspectQueryOptions}; -let db = MonocleDatabase::open_in_dir("~/.monocle")?; -let lens = InspectLens::new(&db); -let result = lens.query("AS13335", &InspectQueryOptions::default())?; +// Start WebSocket server on default port +start_server("127.0.0.1:3000").await?; ``` -#### Scenario 5: CLI Binary (Default) +#### Scenario 3: CLI Binary (Default) **Features**: `cli` (default) The full CLI binary with all features, WebSocket server, and terminal UI: @@ -430,6 +393,12 @@ The full CLI binary with all features, WebSocket server, and terminal UI: monocle = "1.0" ``` +Or explicitly: + +```toml +monocle = { version = "1.0", features = ["cli"] } +``` + ### Valid Feature Combinations All of these combinations compile successfully: @@ -437,23 +406,16 @@ All of these combinations compile successfully: | Combination | Use Case | |-------------|----------| | (none) | Config types only, no functionality | -| `database` | Database access without lenses | -| `lens-core` | Time utilities + database | -| `lens-bgpkit` | Full BGP operations | -| `lens-full` | All lenses | -| `display` | Table formatting only (rarely used alone) | -| `database,display` | Database + table output | -| `lens-core,display` | Time utilities + table output | +| `lib` | Full library functionality | +| `server` | Library + WebSocket server | | `cli` | Full CLI (includes everything) | ### Feature Dependencies When you enable a higher-tier feature, lower-tier features are automatically included: -- `lens-core` → automatically enables `database` -- `lens-bgpkit` → automatically enables `lens-core`, `database`, `display` -- `lens-full` → automatically enables `lens-bgpkit` -- `cli` → automatically enables `lens-full`, `display` +- `server` → automatically enables `lib` +- `cli` → automatically enables `lib` and `server` ## Related Documents diff --git a/CHANGELOG.md b/CHANGELOG.md index 608824c..fe05ae0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,62 @@ All notable changes to this project will be documented in this file. ## Unreleased changes +### Breaking Changes + +* **CLI flag renamed**: `--no-refresh` renamed to `--no-update` for consistency with "update" terminology + * Old: `monocle --no-refresh ` + * New: `monocle --no-update ` + +* **Config subcommands renamed**: Removed `db-` prefix from config subcommands for cleaner syntax + * `monocle config db-refresh` → `monocle config update` + * `monocle config db-backup` → `monocle config backup` + * `monocle config db-sources` → `monocle config sources` + +* **Configurable TTL for all data sources**: All data sources now have configurable cache TTL with 7-day default + * Added `asinfo_cache_ttl_secs` config option (default: 7 days) + * Added `as2rel_cache_ttl_secs` config option (default: 7 days) + * Changed `rpki_cache_ttl_secs` default from 1 hour to 7 days + * Changed `pfx2as_cache_ttl_secs` default from 24 hours to 7 days + * Configure via `~/.monocle/monocle.toml` or environment variables (`MONOCLE_ASINFO_CACHE_TTL_SECS`, etc.) + +* **Simplified feature flags**: Replaced 6-tier feature system with 3 clear features + * Old: `database`, `lens-core`, `lens-bgpkit`, `lens-full`, `display`, `cli` + * New: `lib`, `server`, `cli` + * Quick guide: + - Need CLI binary? Use `cli` (includes everything) + - Need WebSocket server without CLI? Use `server` (includes lib) + - Need only library/data access? Use `lib` (database + all lenses + display) + * Display (tabled) now always included with `lib` feature + +* **Standardized database refresh API**: Consistent interface for all data sources + * New `RefreshResult` struct with `records_loaded`, `source`, `timestamp`, `details` + * Renamed methods for consistency: + - `bootstrap_asinfo()` → `refresh_asinfo()` (with deprecated alias) + - `update_as2rel()` → `refresh_as2rel()` (with deprecated alias) + * Added missing methods: + - `refresh_asinfo_from(path)` - Load ASInfo from custom path + - `refresh_rpki()` - Load RPKI data from records + - `refresh_pfx2as()` - Load Pfx2as data from records + * All repositories now use consistent `needs_*_refresh(ttl)` pattern + * Removed hardcoded TTL methods (`should_update()` from AS2Rel) + * All repositories have both URL and path loading methods + +* **Reorganized examples**: One example per lens with `_lens` suffix + * Flat directory structure: `examples/time_lens.rs`, `examples/rpki_lens.rs`, etc. + * Added new examples for IpLens, Pfx2asLens, As2relLens + * Removed verbose multi-example files + * All examples use `lib` feature exclusively + +### New Features + +* **`monocle config sources`**: Shows staleness status based on TTL for all data sources + * "Stale" column shows whether each source needs updating based on its configured TTL + * Configuration section shows current TTL values for all sources + +### Bug Fixes + +* Avoid creating a new SQLite database when `monocle config sources` inspects staleness + ### Code Improvements * **Data refresh logging**: CLI now shows specific reason for data refresh ("data is empty" vs "data is outdated") instead of generic "empty or outdated" message diff --git a/Cargo.toml b/Cargo.toml index c422906..34ef6fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,133 +17,96 @@ name = "monocle" path = "src/bin/monocle.rs" required-features = ["cli"] -# Existing examples (require lens-bgpkit) -[[example]] -name = "search_bgp_messages" -path = "examples/bgpkit/search_bgp_messages.rs" -required-features = ["lens-bgpkit"] - -[[example]] -name = "ws_client_all" -path = "examples/ws_client_all.rs" -required-features = ["cli"] - # ============================================================================= -# Examples organized by feature requirements +# Examples - One per lens # ============================================================================= -# Standalone examples - minimal dependencies (no database, no bgpkit-*) [[example]] -name = "time_parsing" -path = "examples/standalone/time_parsing.rs" -required-features = ["lens-core"] +name = "time_lens" +path = "examples/time_lens.rs" +required-features = ["lib"] [[example]] -name = "output_formats" -path = "examples/standalone/output_formats.rs" -required-features = ["lens-core"] +name = "country_lens" +path = "examples/country_lens.rs" +required-features = ["lib"] -# Database examples - SQLite operations only [[example]] -name = "database_basics" -path = "examples/database/database_basics.rs" -required-features = ["database"] +name = "ip_lens" +path = "examples/ip_lens.rs" +required-features = ["lib"] [[example]] -name = "as2rel_queries" -path = "examples/database/as2rel_queries.rs" -required-features = ["database"] +name = "parse_lens" +path = "examples/parse_lens.rs" +required-features = ["lib"] [[example]] -name = "pfx2as_search" -path = "examples/database/pfx2as_search.rs" -required-features = ["lens-bgpkit"] +name = "search_lens" +path = "examples/search_lens.rs" +required-features = ["lib"] -# BGPKIT examples - requires bgpkit-* crates [[example]] -name = "country_lookup" -path = "examples/bgpkit/country_lookup.rs" -required-features = ["lens-bgpkit"] +name = "rpki_lens" +path = "examples/rpki_lens.rs" +required-features = ["lib"] [[example]] -name = "rpki_validation" -path = "examples/bgpkit/rpki_validation.rs" -required-features = ["lens-bgpkit"] +name = "pfx2as_lens" +path = "examples/pfx2as_lens.rs" +required-features = ["lib"] [[example]] -name = "mrt_parsing" -path = "examples/bgpkit/mrt_parsing.rs" -required-features = ["lens-bgpkit"] +name = "as2rel_lens" +path = "examples/as2rel_lens.rs" +required-features = ["lib"] -# Full examples - all features [[example]] -name = "inspect_unified" -path = "examples/full/inspect_unified.rs" -required-features = ["lens-full"] +name = "inspect_lens" +path = "examples/inspect_lens.rs" +required-features = ["lib"] [[example]] -name = "progress_callbacks" -path = "examples/full/progress_callbacks.rs" -required-features = ["lens-full"] +name = "database" +path = "examples/database.rs" +required-features = ["lib"] + +[[example]] +name = "ws_client_all" +path = "examples/ws_client_all.rs" +required-features = ["server"] [features] default = ["cli"] # ============================================================================= -# Core feature tiers (layered dependencies) +# Feature flags # ============================================================================= -# Database layer - SQLite operations only -# Minimal dependencies for just database access -# Note: oneio is needed for loading data from URLs in the database layer -database = ["dep:oneio", "dep:ipnet"] - -# Core lens layer - standalone lenses that don't need bgpkit-* -# Includes: TimeLens, OutputFormat utilities -# Note: includes database feature for common usage patterns -lens-core = [ - "database", +# Library feature - includes all database operations, lenses, and display +lib = [ + # Database + "dep:oneio", + "dep:ipnet", + # Lenses "dep:chrono-humanize", "dep:dateparser", "dep:humantime", -] - -# BGPKIT lens layer - lenses requiring bgpkit-* crates -# Includes: CountryLens, ParseLens, SearchLens, IpLens, RpkiLens (commons), Pfx2asLens -# Note: includes display for table formatting (tabled is lightweight) -lens-bgpkit = [ - "lens-core", - "database", - "display", "dep:bgpkit-broker", "dep:bgpkit-parser", "dep:bgpkit-commons", "dep:itertools", "dep:radar-rs", "dep:rayon", + # Display (always included with lib) + "dep:tabled", + "dep:json_to_table", ] -# Full lens layer - all lenses including InspectLens -lens-full = [ - "lens-bgpkit", -] - -# Display support - tabled integration for pretty output -display = ["dep:tabled", "dep:json_to_table"] - -# ============================================================================= -# CLI and server features -# ============================================================================= - -# CLI support (clap derives, terminal output, progress bars) + WebSocket server -cli = [ - "lens-full", - "display", - "dep:clap", - "dep:indicatif", - "dep:tracing-subscriber", - "dep:libc", - # server deps +# WebSocket server feature - for programmatic API access +# Can be used standalone or with CLI +server = [ + "lib", "dep:axum", "dep:tokio", "dep:tokio-util", @@ -153,7 +116,18 @@ cli = [ "dep:tower-http", ] -# Full build with all features (alias for cli) +# CLI feature - full binary with all functionality +# Includes lib and server features +cli = [ + "lib", + "server", + "dep:clap", + "dep:indicatif", + "dep:tracing-subscriber", + "dep:libc", +] + +# Full build (backward compatibility alias) full = ["cli"] [dependencies] @@ -171,21 +145,17 @@ serde_json = "1.0" tracing = "0.1" # ============================================================================= -# Database dependencies (optional but commonly needed) +# Library dependencies (enabled by "lib" feature) # ============================================================================= + +# Database ipnet = { version = "2.10", features = ["json"], optional = true } oneio = { version = "0.20.1", default-features = false, features = ["https", "gz", "bz", "json"], optional = true } -# ============================================================================= -# Lens-core dependencies (optional) -# ============================================================================= +# Lenses chrono-humanize = { version = "0.2", optional = true } dateparser = { version = "0.2", optional = true } humantime = { version = "2.1", optional = true } - -# ============================================================================= -# Lens-bgpkit dependencies (optional) -# ============================================================================= bgpkit-broker = { version = "0.10.1", optional = true } bgpkit-parser = { version = "0.15.0", features = ["serde"], optional = true } bgpkit-commons = { version = "0.10.1", features = ["asinfo", "rpki", "countries"], optional = true } @@ -193,22 +163,12 @@ itertools = { version = "0.14", optional = true } radar-rs = { version = "0.1.0", optional = true } rayon = { version = "1.8", optional = true } -# ============================================================================= -# Display dependencies (optional) -# ============================================================================= +# Display tabled = { version = "0.20", optional = true } json_to_table = { version = "0.12.0", optional = true } # ============================================================================= -# CLI-only dependencies (optional) -# ============================================================================= -clap = { version = "4.1", features = ["derive"], optional = true } -libc = { version = "0.2", optional = true } -indicatif = { version = "0.18.0", optional = true } -tracing-subscriber = { version = "0.3", optional = true } - -# ============================================================================= -# WebSocket server dependencies (optional, included in cli) +# Server dependencies (enabled by "server" feature) # ============================================================================= axum = { version = "0.7", features = ["ws"], optional = true } tokio = { version = "1", features = ["full"], optional = true } @@ -218,6 +178,14 @@ uuid = { version = "1.0", features = ["v4"], optional = true } async-trait = { version = "0.1", optional = true } tower-http = { version = "0.5", features = ["cors", "trace"], optional = true } +# ============================================================================= +# CLI-only dependencies (enabled by "cli" feature) +# ============================================================================= +clap = { version = "4.1", features = ["derive"], optional = true } +libc = { version = "0.2", optional = true } +indicatif = { version = "0.18.0", optional = true } +tracing-subscriber = { version = "0.3", optional = true } + [dev-dependencies] bgpkit-parser = { version = "0.15.0", features = ["serde"] } tempfile = "3" diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index f3a8ced..af568fc 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -70,7 +70,7 @@ src/ ├── monocle.rs # CLI entry point └── commands/ # CLI command handlers ├── as2rel.rs - ├── config.rs # Config display + db-refresh, db-backup, db-sources + ├── config.rs # Config display + update, backup, sources ├── country.rs ├── inspect.rs # Unified AS/prefix inspection (replaces whois, pfx2as) ├── ip.rs diff --git a/README.md b/README.md index b1b08a6..8c2f1fc 100644 --- a/README.md +++ b/README.md @@ -1064,10 +1064,10 @@ Show monocle configuration, data paths, and database management Usage: monocle config [OPTIONS] [COMMAND] Commands: - db-refresh Refresh data source(s) - db-backup Backup the database to a destination - db-sources List available data sources and their status - help Print this message or the help of the given subcommand(s) + update Update data source(s) + backup Backup the database to a destination + sources List available data sources and their status + help Print this message or the help of the given subcommand(s) Options: --debug Print debug information @@ -1093,18 +1093,18 @@ SQLite Database: ~/.monocle/monocle-data.sqlite3 RPKI: 784188 ROAs, 388 ASPAs (updated 2 hours ago) Pfx2as: 1000000 prefixes -# Refresh all data sources -➜ monocle config db-refresh --all +# Update all data sources +➜ monocle config update -# Refresh a specific source -➜ monocle config db-refresh asinfo -➜ monocle config db-refresh rpki +# Update a specific source +➜ monocle config update --asinfo +➜ monocle config update --rpki # Backup the database -➜ monocle config db-backup ~/monocle-backup.sqlite3 +➜ monocle config backup ~/monocle-backup.sqlite3 # List available data sources -➜ monocle config db-sources +➜ monocle config sources ``` ### `monocle server` diff --git a/examples/README.md b/examples/README.md index e6897b5..a021ea2 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,156 +1,58 @@ # Monocle Examples -This directory contains examples demonstrating how to use monocle as a library. -Examples are organized by the features they require, helping you understand -the minimum dependencies needed for different use cases. +Practical examples demonstrating monocle's lens-based API. Each lens has one example showing its primary use case. -## Running All Examples - -You can run these examples using `cargo run --example `. +## Quick Start ```bash -# Standalone utilities -cargo run --release --example time_parsing --features lens-core -cargo run --release --example output_formats --features lens-core - -# Database operations -cargo run --release --example database_basics --features database -cargo run --release --example as2rel_queries --features database -cargo run --release --example pfx2as_search --features lens-bgpkit - -# BGP operations -cargo run --release --example country_lookup --features lens-bgpkit -cargo run --release --example rpki_validation --features lens-bgpkit -cargo run --release --example mrt_parsing --features lens-bgpkit -cargo run --release --example search_bgp_messages --features lens-bgpkit - -# Full functionality -cargo run --release --example inspect_unified --features lens-full -cargo run --release --example progress_callbacks --features lens-full - -# WebSocket Client (requires running server) -# cargo run --example ws_client_all --features cli +cargo run --example --features lib ``` -## Feature Tiers - -Monocle uses a layered feature system: - -| Feature | Description | Key Dependencies | -|---------|-------------|------------------| -| `database` | SQLite database operations | `rusqlite`, `oneio`, `ipnet` | -| `lens-core` | Standalone utilities (TimeLens) | `chrono-humanize`, `dateparser` | -| `lens-bgpkit` | BGP-related lenses | `bgpkit-*`, `rayon`, `tabled` | -| `lens-full` | All lenses including InspectLens | All above | -| `display` | Table formatting | `tabled` (included in lens-bgpkit) | -| `cli` | Full CLI binary | All above + `clap`, `axum` | - -## Examples by Feature - -### Standalone Examples (`lens-core`) - -Minimal dependencies - no bgpkit-* crates required. - -**Files:** -- `standalone/time_parsing.rs` - Parse timestamps, convert formats -- `standalone/output_formats.rs` - Work with OutputFormat enum - -### Database Examples (`database`) - -SQLite operations without lens overhead. - -**Files:** -- `database/database_basics.rs` - MonocleDatabase, schema management -- `database/as2rel_queries.rs` - Query AS-level relationships -- `database/pfx2as_search.rs` - Prefix-to-ASN mapping and search (requires `lens-bgpkit`) +## Lens Examples -### BGPKIT Examples (`lens-bgpkit`) +| Example | Lens | Description | +|---------|------|-------------| +| `time_lens` | TimeLens | Parse timestamps from various formats | +| `country_lens` | CountryLens | Country code/name lookup | +| `ip_lens` | IpLens | IP address information (ASN, RPKI, geolocation) | +| `parse_lens` | ParseLens | Parse MRT files with filters | +| `search_lens` | SearchLens | Search BGP messages via broker | +| `rpki_lens` | RpkiLens | RPKI validation for prefixes | +| `pfx2as_lens` | Pfx2asLens | Prefix-to-ASN mapping lookups | +| `as2rel_lens` | As2relLens | AS-level relationship queries | +| `inspect_lens` | InspectLens | Unified AS/prefix inspection | -Full BGP functionality with bgpkit-* integration. +## Other Examples -**Files:** -- `bgpkit/country_lookup.rs` - Country code/name lookup -- `bgpkit/rpki_validation.rs` - RPKI ROA validation -- `bgpkit/mrt_parsing.rs` - Parse MRT files with filters -- `bgpkit/search_bgp_messages.rs` - Search BGP announcement messages (Real-world example) +| Example | Description | +|---------|-------------| +| `database` | Low-level database operations | +| `ws_client_all` | WebSocket client demo | -### Full Examples (`lens-full`) +## Usage -All lenses including unified inspection. +All examples use the `lib` feature: -**Files:** -- `full/inspect_unified.rs` - InspectLens for unified lookups -- `full/progress_callbacks.rs` - Progress tracking for GUI/CLI - -### CLI/Server Examples (`cli`) - -Examples requiring the full CLI/Server feature set. - -**Files:** -- `ws_client_all.rs` - WebSocket client demonstrating all API methods - -## Using in Your Project - -See the main [README](../README.md) for dependency configuration with version numbers and feature tiers. - -### Minimal Database Access - -```rust -use monocle::database::MonocleDatabase; - -let db = MonocleDatabase::open_in_dir("~/.monocle")?; -if db.needs_as2rel_update() { - db.update_as2rel()?; -} -let rels = db.as2rel().search_asn(13335)?; -``` +```bash +# Time parsing +cargo run --example time_lens --features lib -### Standalone Utilities +# RPKI validation +cargo run --example rpki_lens --features lib -```rust -use monocle::lens::time::{TimeLens, TimeParseArgs}; - -let lens = TimeLens::new(); -let args = TimeParseArgs::new(vec!["2024-01-01T00:00:00Z".to_string()]); -let results = lens.parse(&args)?; +# Unified inspection +cargo run --example inspect_lens --features lib ``` -### BGP Operations +## Common Pattern ```rust use monocle::database::MonocleDatabase; -use monocle::lens::rpki::RpkiLens; +use monocle::lens::rpki::{RpkiLens, RpkiValidationArgs}; let db = MonocleDatabase::open_in_dir("~/.monocle")?; -let mut lens = RpkiLens::new(&db); - -if lens.needs_refresh()? { - lens.refresh()?; -} - +let lens = RpkiLens::new(&db); let result = lens.validate("1.1.1.0/24", 13335)?; -println!("{}: {}", result.state, result.reason); -``` - -### Full Functionality - -```rust -use monocle::database::MonocleDatabase; -use monocle::lens::inspect::{InspectLens, InspectQueryOptions}; - -let db = MonocleDatabase::open_in_dir("~/.monocle")?; -let lens = InspectLens::new(&db); - -lens.ensure_data_available()?; - -let options = InspectQueryOptions::default(); -let results = lens.query(&["13335".to_string()], &options)?; -let json = lens.format_json(&results, true); ``` -## Notes - -- Examples with network operations (RPKI, search) require internet access -- First run may take time to download/bootstrap data -- Use `--release` for better performance with large datasets -- Database operations use WAL mode for concurrent access +See individual example files for complete working code. diff --git a/examples/as2rel_lens.rs b/examples/as2rel_lens.rs new file mode 100644 index 0000000..a167278 --- /dev/null +++ b/examples/as2rel_lens.rs @@ -0,0 +1,39 @@ +//! AS Relationships Example +//! +//! Demonstrates querying AS-level relationships (upstream/downstream/peer). +//! +//! # Running +//! +//! ```bash +//! cargo run --example as2rel_lens --features lib +//! ``` + +use monocle::database::MonocleDatabase; +use monocle::lens::as2rel::{As2relLens, As2relSearchArgs}; + +fn main() -> anyhow::Result<()> { + let db = MonocleDatabase::open_in_memory()?; + let lens = As2relLens::new(&db); + + // Update data if needed + if lens.needs_update() { + println!("Updating AS2Rel data..."); + lens.update()?; + } + + // Search for AS relationships + println!("\nSearching for AS13335 relationships:"); + let args = As2relSearchArgs::new(13335).with_names().upstream_only(); + + let results = lens.search(&args)?; + + println!("Found {} downstream relationships:", results.len()); + for r in results.iter().take(5) { + println!( + " AS{} -> AS{} ({} peers see this)", + r.asn1, r.asn2, r.connected + ); + } + + Ok(()) +} diff --git a/examples/bgpkit/country_lookup.rs b/examples/bgpkit/country_lookup.rs deleted file mode 100644 index e6096d6..0000000 --- a/examples/bgpkit/country_lookup.rs +++ /dev/null @@ -1,173 +0,0 @@ -//! Country Lookup Example -//! -//! This example demonstrates using CountryLens for looking up country -//! codes and names using data from bgpkit-commons. -//! -//! # Feature Requirements -//! -//! This example requires the `lens-bgpkit` feature, which includes -//! bgpkit-commons for country data. -//! -//! # Running -//! -//! ```bash -//! cargo run --example country_lookup --features lens-bgpkit -//! ``` - -use monocle::lens::country::{CountryEntry, CountryLens, CountryLookupArgs, CountryOutputFormat}; - -fn main() -> anyhow::Result<()> { - println!("=== Monocle Country Lookup Example ===\n"); - - let lens = CountryLens::new(); - - // Example 1: Look up by country code - println!("1. Look up by country code (US):"); - let args = CountryLookupArgs::new("US"); - let results = lens.search(&args)?; - print_results(&results); - - // Example 2: Look up by country code (lowercase) - println!("\n2. Look up by country code (lowercase 'de'):"); - let args = CountryLookupArgs::new("de"); - let results = lens.search(&args)?; - print_results(&results); - - // Example 3: Search by partial name - println!("\n3. Search by partial name ('united'):"); - let args = CountryLookupArgs::new("united"); - let results = lens.search(&args)?; - print_results(&results); - - // Example 4: Search by partial name (multiple matches) - println!("\n4. Search by partial name ('island'):"); - let args = CountryLookupArgs::new("island"); - let results = lens.search(&args)?; - print_results(&results); - - // Example 5: Direct lookup methods - println!("\n5. Direct lookup methods:"); - - // Lookup code to get name - if let Some(name) = lens.lookup_code("JP") { - println!(" JP -> {}", name); - } - - if let Some(name) = lens.lookup_code("BR") { - println!(" BR -> {}", name); - } - - // Non-existent code - let result = lens.lookup_code("XX"); - println!(" XX -> {:?}", result); - - // Example 6: List all countries - println!("\n6. List all countries (first 10):"); - let all = lens.all(); - println!(" Total countries: {}", all.len()); - for country in all.iter().take(10) { - println!(" {} - {}", country.code, country.name); - } - println!(" ... and {} more", all.len().saturating_sub(10)); - - // Example 7: Using all_countries() args - println!("\n7. Using all_countries() args:"); - let args = CountryLookupArgs::all_countries(); - let results = lens.search(&args)?; - println!(" Found {} countries using all_countries()", results.len()); - - // Example 8: Different output formats - println!("\n8. Different output formats:"); - - let args = CountryLookupArgs::new("scan"); // Should find Scandinavian countries - let results = lens.search(&args)?; - - println!(" Simple format:"); - println!( - "{}", - lens.format_results(&results, &CountryOutputFormat::Simple) - ); - - println!("\n JSON format:"); - println!( - "{}", - lens.format_results(&results, &CountryOutputFormat::Json) - ); - - // Example 9: Using format_json convenience method - println!("\n9. Using format_json for API responses:"); - let args = CountryLookupArgs::new("AU"); - let results = lens.search(&args)?; - - println!(" Compact JSON:"); - println!(" {}", lens.format_json(&results, false)); - - println!(" Pretty JSON:"); - println!("{}", lens.format_json(&results, true)); - - // Example 10: Builder pattern for args - println!("\n10. Builder pattern for args:"); - let args = CountryLookupArgs::new("FR").with_format(CountryOutputFormat::Simple); - let results = lens.search(&args)?; - println!(" {}", lens.format_results(&results, &args.format)); - - // Example 11: Validation - println!("\n11. Argument validation:"); - - // Valid args - let args = CountryLookupArgs::new("US"); - match args.validate() { - Ok(()) => println!(" Args with query: valid"), - Err(e) => println!(" Args with query: invalid - {}", e), - } - - // Valid args (all flag) - let args = CountryLookupArgs::all_countries(); - match args.validate() { - Ok(()) => println!(" Args with all flag: valid"), - Err(e) => println!(" Args with all flag: invalid - {}", e), - } - - // Invalid args (no query, no all flag) - let args = CountryLookupArgs::default(); - match args.validate() { - Ok(()) => println!(" Empty args: valid"), - Err(e) => println!(" Empty args: invalid - {}", e), - } - - // Example 12: Handle empty results - println!("\n12. Handling empty results:"); - let args = CountryLookupArgs::new("xyznonexistent"); - let results = lens.search(&args)?; - println!(" Results count: {}", results.len()); - println!( - " JSON output: {}", - lens.format_results(&results, &CountryOutputFormat::Json) - ); - println!( - " Simple output: {}", - lens.format_results(&results, &CountryOutputFormat::Simple) - ); - - // Example 13: Common BGP use case - mapping AS country to full name - println!("\n13. Common BGP use case - AS country lookup:"); - let country_codes = ["US", "DE", "JP", "BR", "AU", "SG"]; - println!(" Resolving AS registration countries:"); - for code in &country_codes { - let name = lens.lookup_code(code).unwrap_or("Unknown"); - println!(" AS registered in {} -> {}", code, name); - } - - println!("\n=== Example Complete ==="); - Ok(()) -} - -fn print_results(results: &[CountryEntry]) { - if results.is_empty() { - println!(" No results found"); - } else { - for country in results { - println!(" {} - {}", country.code, country.name); - } - } -} diff --git a/examples/bgpkit/mrt_parsing.rs b/examples/bgpkit/mrt_parsing.rs deleted file mode 100644 index 7f6a00d..0000000 --- a/examples/bgpkit/mrt_parsing.rs +++ /dev/null @@ -1,262 +0,0 @@ -//! MRT Parsing Example -//! -//! This example demonstrates using ParseLens for parsing MRT (Multi-Threaded -//! Routing Toolkit) files, which contain BGP routing data. -//! -//! # Feature Requirements -//! -//! This example requires the `lens-bgpkit` feature, which includes -//! bgpkit-parser for MRT file parsing. -//! -//! # Running -//! -//! ```bash -//! cargo run --example mrt_parsing --features lens-bgpkit -//! ``` -//! -//! Note: This example uses a sample MRT file from the internet. -//! Make sure you have network access. - -use monocle::lens::parse::{ParseElemType, ParseFilters, ParseLens, ParseProgress}; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; - -fn main() -> anyhow::Result<()> { - println!("=== Monocle MRT Parsing Example ===\n"); - - let lens = ParseLens::new(); - - // Example 1: Understanding ParseFilters - println!("1. ParseFilters options:"); - println!(" Available filter fields:"); - println!(" - origin_asn: Filter by origin AS number"); - println!(" - prefix: Filter by network prefix"); - println!(" - include_super: Include super-prefixes when filtering"); - println!(" - include_sub: Include sub-prefixes when filtering"); - println!(" - peer_ip: Filter by peer IP address(es)"); - println!(" - peer_asn: Filter by peer AS number"); - println!(" - elem_type: Filter by announcement (A) or withdrawal (W)"); - println!(" - start_ts: Filter by start timestamp"); - println!(" - end_ts: Filter by end timestamp"); - println!(" - as_path: Filter by AS path regex"); - - // Example 2: Creating filters - println!("\n2. Creating filters:"); - - // Filter for a specific origin ASN - let filters = ParseFilters { - origin_asn: vec!["13335".to_string()], - ..Default::default() - }; - println!(" Filter by origin ASN 13335: {:?}", filters.origin_asn); - - // Filter for a specific prefix - let filters = ParseFilters { - prefix: vec!["1.1.1.0/24".to_string()], - ..Default::default() - }; - println!(" Filter by prefix: {:?}", filters.prefix); - - // Filter for announcements only - let filters = ParseFilters { - elem_type: Some(ParseElemType::A), - ..Default::default() - }; - println!(" Filter announcements only: {:?}", filters.elem_type); - - // Combined filters - let filters = ParseFilters { - origin_asn: vec!["15169".to_string()], - elem_type: Some(ParseElemType::A), - ..Default::default() - }; - println!( - " Combined: origin_asn={:?}, elem_type={:?}", - filters.origin_asn, filters.elem_type - ); - - // Example 3: Filter validation - println!("\n3. Filter validation:"); - let filters = ParseFilters::default(); - match filters.validate() { - Ok(()) => println!(" Default filters: valid"), - Err(e) => println!(" Default filters: invalid - {}", e), - } - - // Example 4: Time-based filtering - println!("\n4. Time-based filtering:"); - let filters = ParseFilters { - start_ts: Some("2024-01-01T00:00:00Z".to_string()), - end_ts: Some("2024-01-01T01:00:00Z".to_string()), - ..Default::default() - }; - println!(" Start: {:?}", filters.start_ts); - println!(" End: {:?}", filters.end_ts); - - // Parse timestamps - match filters.parse_start_end_strings() { - Ok((start, end)) => { - println!(" Parsed start timestamp: {}", start); - println!(" Parsed end timestamp: {}", end); - } - Err(e) => println!(" Parse error: {}", e), - } - - // Example 5: Duration-based filtering - println!("\n5. Duration-based filtering:"); - let filters = ParseFilters { - start_ts: Some("2024-01-01T00:00:00Z".to_string()), - duration: Some("1h".to_string()), - ..Default::default() - }; - println!(" Start: {:?}", filters.start_ts); - println!(" Duration: {:?}", filters.duration); - - // Example 6: AS path regex filtering - println!("\n6. AS path regex filtering:"); - let filters = ParseFilters { - as_path: Some("13335$".to_string()), // Paths ending with AS13335 - ..Default::default() - }; - println!(" AS path regex: {:?}", filters.as_path); - println!(" This matches paths where AS13335 is the origin"); - - // Example 7: Progress callback - println!("\n7. Progress callback setup:"); - let message_count = Arc::new(AtomicU64::new(0)); - let count_clone = message_count.clone(); - - let callback = Arc::new(move |progress: ParseProgress| match progress { - ParseProgress::Started { file_path } => { - println!(" Started parsing: {}", file_path); - } - ParseProgress::Update { - messages_processed, - rate, - elapsed_secs, - } => { - let rate_str = rate - .map(|r| format!("{:.0} msg/s", r)) - .unwrap_or_else(|| "N/A".to_string()); - println!( - " Progress: {} messages, {}, {:.1}s elapsed", - messages_processed, rate_str, elapsed_secs - ); - } - ParseProgress::Completed { - total_messages, - duration_secs, - rate, - } => { - count_clone.store(total_messages, Ordering::SeqCst); - let rate_str = rate - .map(|r| format!("{:.0} msg/s", r)) - .unwrap_or_else(|| "N/A".to_string()); - println!( - " Completed: {} messages in {:.2}s ({})", - total_messages, duration_secs, rate_str - ); - } - }); - - println!(" Callback created (would be used with parse_with_progress)"); - - // Example 8: Parsing a remote MRT file (demonstration) - println!("\n8. Parsing demonstration:"); - println!(" Parsing a remote MRT file..."); - - let filters = ParseFilters { - origin_asn: vec!["13335".to_string()], - ..Default::default() - }; - let url = "https://data.ris.ripe.net/rrc00/2024.01/updates.20240101.0000.gz"; - let elems = lens.parse_with_progress(&filters, url, Some(callback))?; - - println!(" Found {} elements:", elems.len()); - for elem in elems.iter().take(5) { - println!(" {:?}", elem); - } - if elems.len() > 5 { - println!(" ... and {} more", elems.len() - 5); - } - - // Example 9: Handler-based parsing (for streaming) - println!("\n9. Handler-based parsing (streaming):"); - println!(" For memory-efficient processing of large files:"); - println!(" ```rust"); - println!(" let handler = Arc::new(|elem: BgpElem| {{"); - println!(" // Process each element as it's parsed"); - println!(" println!(\"Prefix: {{}}\", elem.prefix);"); - println!(" }});"); - println!(" lens.parse_with_handler(&filters, url, handler)?;"); - println!(" ```"); - - // Example 10: Creating parser directly - println!("\n10. Creating parser directly:"); - println!(" For more control, create a BgpkitParser directly:"); - println!(" ```rust"); - println!(" let parser = lens.create_parser(&filters, \"path/to/file.mrt\")?;"); - println!(" for elem in parser {{"); - println!(" // Process elements"); - println!(" }}"); - println!(" ```"); - - // Example 11: Common use cases - println!("\n11. Common use cases:"); - - println!("\n a) Find all announcements for a prefix:"); - let _filters = ParseFilters { - prefix: vec!["8.8.8.0/24".to_string()], - elem_type: Some(ParseElemType::A), - ..Default::default() - }; - - println!("\n b) Find withdrawals from a specific peer:"); - let _filters = ParseFilters { - peer_asn: vec!["174".to_string()], - elem_type: Some(ParseElemType::W), - ..Default::default() - }; - - println!("\n c) Find routes with a specific AS in path:"); - let _filters = ParseFilters { - as_path: Some(".*13335.*".to_string()), - ..Default::default() - }; - - println!("\n d) Find routes originated by an AS:"); - let _filters = ParseFilters { - origin_asn: vec!["13335".to_string()], - ..Default::default() - }; - - // Example 12: Element types - println!("\n12. BGP element types:"); - println!(" ParseElemType::A - Announcement (route advertisement)"); - println!(" ParseElemType::W - Withdrawal (route removal)"); - println!( - " Display: A = '{}', W = '{}'", - ParseElemType::A, - ParseElemType::W - ); - - // Example 13: Best practices - println!("\n13. Best practices:"); - println!(" - Use filters to reduce memory usage and processing time"); - println!(" - Use parse_with_handler() for very large files"); - println!(" - Use progress callbacks for user feedback on long operations"); - println!(" - Validate filters before parsing"); - println!(" - Consider time-based filtering for RIB dumps"); - println!(" - Use origin_asn filter for targeted analysis"); - - // Example 14: Supported file formats - println!("\n14. Supported file formats:"); - println!(" - Raw MRT files (.mrt)"); - println!(" - Gzip compressed (.gz)"); - println!(" - Bzip2 compressed (.bz2)"); - println!(" - Remote URLs (http://, https://)"); - println!(" - Local file paths"); - - println!("\n=== Example Complete ==="); - Ok(()) -} diff --git a/examples/bgpkit/rpki_validation.rs b/examples/bgpkit/rpki_validation.rs deleted file mode 100644 index 703b94e..0000000 --- a/examples/bgpkit/rpki_validation.rs +++ /dev/null @@ -1,241 +0,0 @@ -//! RPKI Validation Example -//! -//! This example demonstrates using RpkiLens for RPKI validation operations -//! including ROA lookups and prefix-ASN validation. -//! -//! # Feature Requirements -//! -//! This example requires the `lens-bgpkit` feature, which includes -//! bgpkit-commons for RPKI data. -//! -//! # Running -//! -//! ```bash -//! cargo run --example rpki_validation --features lens-bgpkit -//! ``` -//! -//! Note: This example requires network access to fetch RPKI data on first run. - -use monocle::database::MonocleDatabase; -use monocle::lens::rpki::{ - RpkiAspaLookupArgs, RpkiDataSource, RpkiLens, RpkiOutputFormat, RpkiRoaLookupArgs, - RpkiValidationState, -}; - -fn main() -> anyhow::Result<()> { - println!("=== Monocle RPKI Validation Example ===\n"); - - // Create an in-memory database for this example - // In production, use MonocleDatabase::open_in_dir("~/.monocle") - let db = MonocleDatabase::open_in_memory()?; - let mut lens = RpkiLens::new(&db); - - // Example 1: Check cache status - println!("1. Cache status:"); - println!(" Cache is empty: {:?}", lens.is_empty()?); - println!(" Needs refresh: {:?}", lens.needs_refresh()?); - - // Example 2: Refresh RPKI cache (fetch from Cloudflare) - println!("\n2. Refreshing RPKI cache from Cloudflare..."); - match lens.refresh() { - Ok((roa_count, aspa_count)) => { - println!(" Loaded {} ROAs and {} ASPAs", roa_count, aspa_count); - } - Err(e) => { - println!(" Warning: Could not refresh cache: {}", e); - println!(" (This may be due to network issues)"); - println!(" Continuing with demonstration of API..."); - } - } - - // Example 3: Get cache metadata - println!("\n3. Cache metadata:"); - if let Ok(Some(meta)) = lens.get_metadata() { - println!(" Updated at: {}", meta.updated_at); - println!(" ROA count: {}", meta.roa_count); - println!(" ASPA count: {}", meta.aspa_count); - } else { - println!(" No metadata available (cache may be empty)"); - } - - // Example 4: Validate prefix-ASN pairs - println!("\n4. RPKI validation examples:"); - - let test_cases = [ - ("1.1.1.0/24", 13335, "Cloudflare"), - ("8.8.8.0/24", 15169, "Google DNS"), - ("1.1.1.0/24", 12345, "Wrong ASN for Cloudflare prefix"), - ("192.0.2.0/24", 64496, "Documentation prefix"), - ]; - - for (prefix, asn, description) in &test_cases { - match lens.validate(prefix, *asn) { - Ok(result) => { - let state_icon = match result.state { - RpkiValidationState::Valid => "✓", - RpkiValidationState::Invalid => "✗", - RpkiValidationState::NotFound => "?", - }; - println!( - " {} {} AS{}: {} - {}", - state_icon, prefix, asn, result.state, description - ); - if !result.covering_roas.is_empty() { - println!(" Covering ROAs: {}", result.covering_roas.len()); - } - } - Err(e) => { - println!(" ! {} AS{}: Error - {} ({})", prefix, asn, e, description); - } - } - } - - // Example 5: Get covering ROAs for a prefix - println!("\n5. Get covering ROAs for a prefix:"); - let prefix = "1.1.1.0/24"; - match lens.get_covering_roas(prefix) { - Ok(roas) => { - println!(" Covering ROAs for {}:", prefix); - if roas.is_empty() { - println!(" No covering ROAs found"); - } else { - for roa in roas.iter().take(5) { - println!( - " {} max:{} AS{} ({})", - roa.prefix, roa.max_length, roa.origin_asn, roa.ta - ); - } - if roas.len() > 5 { - println!(" ... and {} more", roas.len() - 5); - } - } - } - Err(e) => println!(" Error: {}", e), - } - - // Example 6: Look up ROAs by ASN - println!("\n6. Look up ROAs by ASN:"); - let args = RpkiRoaLookupArgs::new().with_asn(13335); - match lens.get_roas(&args) { - Ok(roas) => { - println!(" ROAs for AS13335 (Cloudflare):"); - if roas.is_empty() { - println!(" No ROAs found (cache may be empty)"); - } else { - for roa in roas.iter().take(10) { - println!(" {} max:{} ({})", roa.prefix, roa.max_length, roa.ta); - } - if roas.len() > 10 { - println!(" ... and {} more", roas.len() - 10); - } - } - } - Err(e) => println!(" Error: {}", e), - } - - // Example 7: Look up ROAs by prefix - println!("\n7. Look up ROAs by prefix:"); - let args = RpkiRoaLookupArgs::new().with_prefix("8.8.8.0/24"); - match lens.get_roas(&args) { - Ok(roas) => { - println!(" ROAs covering 8.8.8.0/24:"); - if roas.is_empty() { - println!(" No ROAs found"); - } else { - for roa in &roas { - println!( - " {} max:{} AS{} ({})", - roa.prefix, roa.max_length, roa.origin_asn, roa.ta - ); - } - } - } - Err(e) => println!(" Error: {}", e), - } - - // Example 8: ASPA lookups - println!("\n8. ASPA (AS Provider Authorization) lookups:"); - let args = RpkiAspaLookupArgs::new().with_customer(13335); - match lens.get_aspas(&args) { - Ok(aspas) => { - println!(" ASPAs for AS13335 as customer:"); - if aspas.is_empty() { - println!(" No ASPAs found"); - } else { - for aspa in &aspas { - println!( - " Customer AS{} authorized providers:", - aspa.customer_asn - ); - let providers: Vec = aspa - .providers - .iter() - .map(|a| format!("AS{}", a.asn)) - .collect(); - println!(" {}", providers.join(", ")); - } - } - } - Err(e) => println!(" Error: {}", e), - } - - // Example 9: Data sources - println!("\n9. Available data sources:"); - println!(" - Cloudflare: Current RPKI data (default)"); - println!(" - RIPE NCC: Historical RPKI data"); - println!(" - RPKIviews: Historical RPKI data with multiple collectors"); - - let args = RpkiRoaLookupArgs::new() - .with_asn(13335) - .with_source(RpkiDataSource::Cloudflare); - println!("\n Query with explicit source:"); - println!(" Source: {:?}", args.source); - println!(" Is historical: {}", args.is_historical()); - - // Example 10: Output formatting - println!("\n10. Output formatting:"); - let args = RpkiRoaLookupArgs::new().with_asn(15169); - match lens.get_roas(&args) { - Ok(roas) => { - if !roas.is_empty() { - let sample = &roas[..roas.len().min(3)]; - - println!(" JSON format:"); - println!("{}", lens.format_roas(sample, &RpkiOutputFormat::Json)); - - println!("\n Pretty format:"); - println!("{}", lens.format_roas(sample, &RpkiOutputFormat::Pretty)); - } else { - println!(" (No ROAs to format - cache may be empty)"); - } - } - Err(e) => println!(" Error: {}", e), - } - - // Example 11: Validation result formatting - println!("\n11. Validation result formatting:"); - if let Ok(result) = lens.validate("1.1.1.0/24", 13335) { - println!(" Table format:"); - println!( - "{}", - lens.format_validation(&result, &RpkiOutputFormat::Table) - ); - - println!(" JSON format:"); - println!( - "{}", - lens.format_validation(&result, &RpkiOutputFormat::Json) - ); - } - - // Example 12: Best practices - println!("\n12. Best practices:"); - println!(" - Check needs_refresh() and refresh cache periodically (24h default TTL)"); - println!(" - Use validate() for single prefix-ASN validation"); - println!(" - Use get_covering_roas() to understand why validation failed"); - println!(" - Cache the RpkiLens instance - it reuses the database connection"); - println!(" - For historical queries, use with_date() and appropriate source"); - - println!("\n=== Example Complete ==="); - Ok(()) -} diff --git a/examples/bgpkit/search_bgp_messages.rs b/examples/bgpkit/search_bgp_messages.rs deleted file mode 100644 index b44e63d..0000000 --- a/examples/bgpkit/search_bgp_messages.rs +++ /dev/null @@ -1,100 +0,0 @@ -//! Example: Search BGP announcement messages -//! -//! This example demonstrates how to use monocle as a library to search for -//! BGP announcement messages from the first hour of 2025 using the rrc00 collector. -//! -//! Run with: cargo run --example search_bgp_messages - -use monocle::lens::parse::ParseFilters; -use monocle::lens::search::{SearchDumpType, SearchFilters}; - -fn main() -> anyhow::Result<()> { - // Create search filters for the first hour of 2025 using rrc00 collector - let filters = SearchFilters { - parse_filters: ParseFilters { - // Time range: 2025-01-01 00:00:00 to 2025-01-01 01:00:00 UTC - start_ts: Some("2025-01-01T00:00:00Z".to_string()), - end_ts: Some("2025-01-01T01:00:00Z".to_string()), - // Optional: filter by origin ASN (e.g., Cloudflare) - // origin_asn: Some(13335), - // Optional: filter by prefix - // prefix: Some("1.1.1.0/24".to_string()), - ..Default::default() - }, - // Use rrc00 collector from RIPE RIS - collector: Some("rrc00".to_string()), - // Only RIPE RIS project - project: Some("riperis".to_string()), - // Only BGP updates (not RIB dumps) - dump_type: SearchDumpType::Updates, - }; - - // Validate filters - filters.validate()?; - - // Build the broker query to find MRT files - let broker = filters.build_broker()?; - - println!("Searching for BGP messages from rrc00 during the first hour of 2025..."); - println!(); - - // Query the broker for available MRT files - let items = broker.query()?; - - println!("Found {} MRT files to process", items.len()); - println!(); - - // Process the first file as a demonstration - if let Some(first_item) = items.first() { - println!( - "Processing first file: {} ({})", - first_item.url, first_item.collector_id - ); - println!( - "Time range: {} - {}", - first_item.ts_start.format("%Y-%m-%d %H:%M:%S UTC"), - first_item.ts_end.format("%Y-%m-%d %H:%M:%S UTC") - ); - println!(); - - // Create a parser for the MRT file with filters applied - let parser = filters.to_parser(&first_item.url)?; - - // Count and display the first few BGP elements - let mut count = 0; - let max_display = 5; - - for elem in parser { - count += 1; - - // Display the first few elements - if count <= max_display { - println!("BGP Element #{}:", count); - println!(" Type: {:?}", elem.elem_type); - println!(" Timestamp: {}", elem.timestamp); - println!(" Peer IP: {}", elem.peer_ip); - println!(" Peer ASN: {}", elem.peer_asn); - println!(" Prefix: {}", elem.prefix); - if let Some(path) = &elem.as_path { - println!(" AS Path: {:?}", path.to_u32_vec_opt(true)); - } - if let Some(origin) = &elem.origin_asns { - let asns: Vec = origin.iter().map(|a| a.to_u32()).collect(); - println!(" Origin ASN(s): {:?}", asns); - } - println!(); - } - - // Stop after processing some messages for the demo - if count >= 100 { - break; - } - } - - println!("Processed {} BGP elements (limited to 100 for demo)", count); - } else { - println!("No MRT files found for the specified time range"); - } - - Ok(()) -} diff --git a/examples/country_lens.rs b/examples/country_lens.rs new file mode 100644 index 0000000..6db1e75 --- /dev/null +++ b/examples/country_lens.rs @@ -0,0 +1,33 @@ +//! Country Lookup Example +//! +//! Demonstrates looking up country codes and names. +//! +//! # Running +//! +//! ```bash +//! cargo run --example country_lookup --features lib +//! ``` + +use monocle::lens::country::{CountryLens, CountryLookupArgs}; + +fn main() -> anyhow::Result<()> { + let lens = CountryLens::new(); + + // Look up by country code + println!("Looking up 'US':"); + let args = CountryLookupArgs::new("US"); + let results = lens.search(&args)?; + for country in &results { + println!(" {} - {}", country.code, country.name); + } + + // Search by partial name + println!("\nSearching for 'united':"); + let args = CountryLookupArgs::new("united"); + let results = lens.search(&args)?; + for country in &results { + println!(" {} - {}", country.code, country.name); + } + + Ok(()) +} diff --git a/examples/database.rs b/examples/database.rs new file mode 100644 index 0000000..57c39f6 --- /dev/null +++ b/examples/database.rs @@ -0,0 +1,36 @@ +//! Database Example +//! +//! Demonstrates basic database operations with MonocleDatabase. +//! +//! # Running +//! +//! ```bash +//! cargo run --example database --features lib +//! ``` + +use monocle::database::MonocleDatabase; +use std::time::Duration; + +fn main() -> anyhow::Result<()> { + // Open an in-memory database + let db = MonocleDatabase::open_in_memory()?; + + // Check repository status + println!("Database Status:"); + println!(" AS2Rel: {} records", db.as2rel().count()?); + println!(" ASInfo: {} records", db.asinfo().core_count()); + + // Check if refresh is needed + let ttl = Duration::from_secs(24 * 60 * 60); + if db.needs_as2rel_refresh(ttl) { + println!("\nAS2Rel data needs refresh (older than 24h)"); + } + + // Query relationships (if data exists) + let rels = db.as2rel().search_asn(13335)?; + if !rels.is_empty() { + println!("\nFound {} relationships for AS13335", rels.len()); + } + + Ok(()) +} diff --git a/examples/database/as2rel_queries.rs b/examples/database/as2rel_queries.rs deleted file mode 100644 index 0e2c271..0000000 --- a/examples/database/as2rel_queries.rs +++ /dev/null @@ -1,138 +0,0 @@ -//! AS2Rel Queries Example -//! -//! This example demonstrates querying AS-level relationship data using -//! the AS2Rel repository directly from the database layer. -//! -//! # Feature Requirements -//! -//! This example only requires the `database` feature, which has minimal -//! dependencies (rusqlite, serde, chrono). -//! -//! Note: To actually query data, the database needs to be populated first. -//! This example shows how to work with the repository API even with an -//! empty database. -//! -//! # Running -//! -//! ```bash -//! cargo run --example as2rel_queries --features database -//! ``` - -use monocle::database::{MonocleDatabase, BGPKIT_AS2REL_URL}; - -fn main() -> anyhow::Result<()> { - println!("=== Monocle AS2Rel Queries Example ===\n"); - - // Create an in-memory database for demonstration - let db = MonocleDatabase::open_in_memory()?; - - // Example 1: Check repository status - println!("1. Repository status:"); - let as2rel = db.as2rel(); - println!(" Repository is empty: {}", as2rel.is_empty()); - println!(" Needs update: {}", db.needs_as2rel_update()); - println!(" Data source URL: {}", BGPKIT_AS2REL_URL); - - // Example 2: Understanding the data model - println!("\n2. AS2Rel data model:"); - println!(" The AS2Rel repository stores AS-level relationships:"); - println!(" - asn1: First AS in the relationship"); - println!(" - asn2: Second AS in the relationship"); - println!(" - rel: Relationship type (-1, 0, 1)"); - println!(" -1 = asn1 is customer of asn2"); - println!(" 0 = peer-to-peer relationship"); - println!(" 1 = asn1 is provider of asn2"); - println!(" - paths_count: Number of AS paths containing this relationship"); - println!(" - peers_count: Number of BGP peers observing this relationship"); - - // Example 3: Metadata operations - println!("\n3. Metadata operations:"); - let max_peers = as2rel.get_max_peers_count(); - println!( - " Max peers count: {} (used for percentage calculations)", - max_peers - ); - - // Example 4: Query API overview (with empty database) - println!("\n4. Query API methods (showing with empty database):"); - - // Search by single ASN - let results = as2rel.search_asn(13335)?; - println!(" search_asn(13335): {} results", results.len()); - - // Search by ASN pair - let results = as2rel.search_pair(13335, 15169)?; - println!(" search_pair(13335, 15169): {} results", results.len()); - - // Search with names (joins with ASInfo if available) - let results = as2rel.search_asn_with_names(13335)?; - println!(" search_asn_with_names(13335): {} results", results.len()); - - // Note: get_connectivity_summary requires additional parameters - // It's typically used through higher-level APIs like InspectLens - println!(" get_connectivity_summary: requires name lookup function (see InspectLens)"); - - // Example 5: Using the repository with populated data - println!("\n5. Working with populated data (simulation):"); - println!(" In a real application, you would:"); - println!(" a) Use MonocleDatabase::open_in_dir(\"~/.monocle\")"); - println!(" b) Call db.update_as2rel() to fetch latest data"); - println!(" c) Query using the methods shown above"); - println!("\n Example code:"); - println!(" ```rust"); - println!(" let db = MonocleDatabase::open_in_dir(\"~/.monocle\")?;"); - println!(" if db.needs_as2rel_update() {{"); - println!(" let count = db.update_as2rel()?;"); - println!(" println!(\"Loaded {{}} relationships\", count);"); - println!(" }}"); - println!(" let rels = db.as2rel().search_asn(13335)?;"); - println!(" ```"); - - // Example 6: Understanding aggregated relationships - println!("\n6. Aggregated relationships:"); - println!(" The search_asn_with_names() method returns AggregatedRelationship:"); - println!(" - asn1, asn2: The AS pair"); - println!(" - asn2_name: Name of asn2 (if available from ASInfo)"); - println!(" - connected_count: Total paths where relationship was observed"); - println!(" - as1_upstream_count: Paths where asn1 appears as upstream"); - println!(" - as2_upstream_count: Paths where asn2 appears as upstream"); - println!("\n This allows calculating:"); - println!(" - Visibility percentage (connected_count / max_peers * 100)"); - println!(" - Relationship direction confidence"); - - // Example 7: Raw repository access - println!("\n7. Accessing raw connection for custom queries:"); - let conn = db.connection(); - - // Check what tables exist - let mut stmt = - conn.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'as2rel%'")?; - let tables: Vec = stmt - .query_map([], |row| row.get(0))? - .filter_map(|r| r.ok()) - .collect(); - - println!(" AS2Rel tables in database:"); - for table in &tables { - println!(" - {}", table); - } - - // Example 8: Best practices - println!("\n8. Best practices:"); - println!(" - Check needs_as2rel_update() before querying to ensure fresh data"); - println!(" - Use search_asn_with_names() to get human-readable results"); - println!(" - Cache max_peers_count for percentage calculations"); - println!(" - For bulk operations, use get_connectivity_summary()"); - println!(" - The database uses WAL mode for concurrent read performance"); - - // Example 9: Error handling patterns - println!("\n9. Error handling:"); - println!(" All repository methods return anyhow::Result"); - println!(" Common errors:"); - println!(" - Database not found (use open_in_dir with valid path)"); - println!(" - Schema mismatch (will auto-migrate on open)"); - println!(" - Empty database (check is_empty() before querying)"); - - println!("\n=== Example Complete ==="); - Ok(()) -} diff --git a/examples/database/database_basics.rs b/examples/database/database_basics.rs deleted file mode 100644 index 30d5725..0000000 --- a/examples/database/database_basics.rs +++ /dev/null @@ -1,141 +0,0 @@ -//! Database Basics Example -//! -//! This example demonstrates using MonocleDatabase for SQLite operations -//! without requiring bgpkit-* dependencies. -//! -//! # Feature Requirements -//! -//! This example only requires the `database` feature, which has minimal -//! dependencies (rusqlite, serde, chrono). -//! -//! # Running -//! -//! ```bash -//! cargo run --example database_basics --features database -//! ``` - -use monocle::database::{DatabaseConn, MonocleDatabase, SchemaManager, SchemaStatus}; - -fn main() -> anyhow::Result<()> { - println!("=== Monocle Database Basics Example ===\n"); - - // Example 1: Create an in-memory database - println!("1. Creating in-memory database:"); - let db = MonocleDatabase::open_in_memory()?; - println!(" Database created successfully"); - - // Example 2: Check database repositories - println!("\n2. Checking database repositories:"); - println!(" AS2Rel is empty: {}", db.as2rel().is_empty()); - println!(" ASInfo is empty: {}", db.asinfo().is_empty()); - println!(" RPKI is empty: {}", db.rpki().is_empty()); - println!(" Pfx2as is empty: {}", db.pfx2as().is_empty()); - - // Example 3: Check if updates are needed - println!("\n3. Checking update status:"); - println!(" Needs ASInfo bootstrap: {}", db.needs_asinfo_bootstrap()); - println!(" Needs AS2Rel update: {}", db.needs_as2rel_update()); - println!(" Needs RPKI refresh: {}", db.needs_rpki_refresh()); - println!(" Needs Pfx2as refresh: {}", db.needs_pfx2as_refresh()); - - // Example 4: Working with metadata - println!("\n4. Working with metadata:"); - db.set_meta("example_key", "example_value")?; - let value = db.get_meta("example_key")?; - println!(" Set 'example_key' to 'example_value'"); - println!(" Retrieved: {:?}", value); - - // Example 5: Using DatabaseConn directly for custom queries - println!("\n5. Using DatabaseConn for custom operations:"); - let conn = DatabaseConn::open_in_memory()?; - - // Create a custom table - conn.execute("CREATE TABLE custom_data (id INTEGER PRIMARY KEY, name TEXT, value REAL)")?; - println!(" Created custom table 'custom_data'"); - - // Check if table exists - let exists = conn.table_exists("custom_data")?; - println!(" Table 'custom_data' exists: {}", exists); - - // Insert some data - conn.execute_with_params( - "INSERT INTO custom_data (name, value) VALUES (?1, ?2)", - ("test_entry", 42.5), - )?; - println!(" Inserted test entry"); - - // Check row count - let count = conn.table_count("custom_data")?; - println!(" Row count: {}", count); - - // Example 6: Schema management - println!("\n6. Schema management:"); - let schema_conn = DatabaseConn::open_in_memory()?; - let schema_mgr = SchemaManager::new(&schema_conn.conn); - - let status = schema_mgr.check_status()?; - println!(" Initial schema status: {:?}", status); - - match status { - SchemaStatus::NotInitialized => { - println!(" Initializing schema..."); - schema_mgr.initialize()?; - println!(" Schema initialized successfully"); - } - SchemaStatus::Current => { - println!(" Schema is already current"); - } - _ => { - println!(" Schema needs attention: {:?}", status); - } - } - - // Example 7: Working with transactions - println!("\n7. Using transactions:"); - let tx_conn = DatabaseConn::open_in_memory()?; - tx_conn.execute("CREATE TABLE tx_test (id INTEGER PRIMARY KEY, data TEXT)")?; - - { - let tx = tx_conn.transaction()?; - - tx.execute("INSERT INTO tx_test (data) VALUES ('item1')", [])?; - tx.execute("INSERT INTO tx_test (data) VALUES ('item2')", [])?; - tx.execute("INSERT INTO tx_test (data) VALUES ('item3')", [])?; - - tx.commit()?; - println!(" Transaction committed successfully"); - } - - let final_count = tx_conn.table_count("tx_test")?; - println!(" Final row count: {}", final_count); - - // Example 8: Accessing raw connection for advanced queries - println!("\n8. Advanced queries with raw connection:"); - let raw_conn = db.connection(); - - let mut stmt = raw_conn.prepare("SELECT name FROM sqlite_master WHERE type='table'")?; - let tables: Vec = stmt - .query_map([], |row| row.get(0))? - .filter_map(|r| r.ok()) - .collect(); - - println!(" Tables in database:"); - for table in &tables { - println!(" - {}", table); - } - - // Example 9: Database file path (persistent database) - println!("\n9. Persistent database example (not actually created):"); - let temp_dir = std::env::temp_dir(); - let db_path = temp_dir.join("monocle-example.sqlite3"); - println!(" Would create database at: {}", db_path.display()); - println!(" Use MonocleDatabase::open(&path) for persistent storage"); - - // Example 10: Using open_in_dir - println!("\n10. Using open_in_dir pattern:"); - println!(" MonocleDatabase::open_in_dir(\"~/.monocle\") creates:"); - println!(" ~/.monocle/monocle-data.sqlite3"); - - println!("\n=== Example Complete ==="); - Ok(()) -} diff --git a/examples/database/pfx2as_search.rs b/examples/database/pfx2as_search.rs deleted file mode 100644 index 0892715..0000000 --- a/examples/database/pfx2as_search.rs +++ /dev/null @@ -1,196 +0,0 @@ -//! Pfx2as Search Example -//! -//! This example demonstrates using the Pfx2asLens for prefix-to-ASN mapping -//! operations, including search by prefix, search by ASN, and RPKI validation. -//! -//! # Feature Requirements -//! -//! This example requires the `lens-bgpkit` feature for full functionality, -//! but the basic database operations work with just `database`. -//! -//! # Running -//! -//! ```bash -//! cargo run --example pfx2as_search --features lens-bgpkit -//! ``` -//! -//! Note: First run may take time to download pfx2as and RPKI data. - -use monocle::database::MonocleDatabase; - -fn main() -> anyhow::Result<()> { - println!("=== Monocle Pfx2as Search Example ===\n"); - - // Example 1: Check repository status with in-memory database - println!("1. Repository status (in-memory database):"); - let db = MonocleDatabase::open_in_memory()?; - let pfx2as = db.pfx2as(); - println!(" Repository is empty: {}", pfx2as.is_empty()); - println!(" Needs refresh: {}", db.needs_pfx2as_refresh()); - - // Example 2: Understanding the data model - println!("\n2. Pfx2as data model:"); - println!(" The Pfx2as repository stores prefix-to-ASN mappings:"); - println!(" - prefix: IP prefix (e.g., '1.1.1.0/24')"); - println!(" - origin_asn: The ASN announcing this prefix"); - println!(" - validation: RPKI validation status ('valid', 'invalid', 'unknown')"); - println!("\n Prefixes are stored with blob-based range queries for efficient lookups:"); - println!(" - Exact match: Find prefixes that exactly match"); - println!(" - Longest match: Find the most specific covering prefix"); - println!(" - Covering: Find all super-prefixes (less specific)"); - println!(" - Covered: Find all sub-prefixes (more specific)"); - - // Example 3: Query API overview (with empty database) - println!("\n3. Low-level query API methods (showing with empty database):"); - - // Exact lookup - let results = pfx2as.lookup_exact("1.1.1.0/24")?; - println!(" lookup_exact('1.1.1.0/24'): {} ASNs", results.len()); - - // Longest prefix match - let result = pfx2as.lookup_longest("1.1.1.1/32")?; - println!( - " lookup_longest('1.1.1.1/32'): {} ASNs", - result.origin_asns.len() - ); - - // Covering prefixes (supernets) - let results = pfx2as.lookup_covering("1.1.1.0/24")?; - println!( - " lookup_covering('1.1.1.0/24'): {} prefixes", - results.len() - ); - - // Covered prefixes (subnets) - let results = pfx2as.lookup_covered("1.0.0.0/8")?; - println!(" lookup_covered('1.0.0.0/8'): {} prefixes", results.len()); - - // Get prefixes by ASN - let records = pfx2as.get_by_asn(13335)?; - println!(" get_by_asn(13335): {} prefixes", records.len()); - - // Example 4: Using the lens for high-level operations - println!("\n4. High-level Pfx2asLens API (requires lens-bgpkit feature):"); - println!(" The Pfx2asLens provides:"); - println!(" - search(): Auto-detect query type (ASN or prefix)"); - println!(" - search_by_asn(): Get all prefixes for an ASN with RPKI validation"); - println!(" - search_by_prefix(): Get origin ASNs with sub/super prefix options"); - println!(" - RPKI validation integration"); - println!(" - AS name enrichment (from ASInfo database)"); - - // Example 5: Working with populated data (simulation) - println!("\n5. Working with populated data:"); - println!(" In a real application, you would:"); - println!(" a) Use MonocleDatabase::open_in_dir(\"~/.monocle\")"); - println!(" b) Check lens.needs_refresh() and call lens.refresh() if needed"); - println!(" c) Use lens.search() for high-level queries"); - - println!("\n Example code (search by prefix):"); - println!(" ```rust"); - println!(" use monocle::database::MonocleDatabase;"); - println!(" use monocle::lens::pfx2as::{{Pfx2asLens, Pfx2asSearchArgs}};"); - println!(); - println!(" let db = MonocleDatabase::open_in_dir(\"~/.monocle\")?;"); - println!(" let lens = Pfx2asLens::new(&db);"); - println!(); - println!(" // Ensure data is available"); - println!(" if lens.needs_refresh()? {{"); - println!(" lens.refresh(None)?;"); - println!(" }}"); - println!(); - println!(" // Search by prefix with RPKI validation and AS names"); - println!(" let args = Pfx2asSearchArgs::new(\"1.1.1.0/24\")"); - println!(" .with_show_name(true);"); - println!(" let results = lens.search(&args)?;"); - println!(); - println!(" for r in &results {{"); - println!(" println!(\"{{}} -> AS{{}} ({{}})\", r.prefix, r.origin_asn, r.rpki);"); - println!(" }}"); - println!(" ```"); - - println!("\n Example code (search by ASN):"); - println!(" ```rust"); - println!(" // Search by ASN - get all prefixes announced by an AS"); - println!(" let args = Pfx2asSearchArgs::new(\"13335\")"); - println!(" .with_show_name(true)"); - println!(" .with_limit(10);"); - println!(" let results = lens.search(&args)?;"); - println!(); - println!(" for r in &results {{"); - println!(" println!(\"{{}} ({{}})\", r.prefix, r.rpki);"); - println!(" }}"); - println!(" ```"); - - println!("\n Example code (include sub/super prefixes):"); - println!(" ```rust"); - println!(" // Search with sub-prefixes (more specific)"); - println!(" let args = Pfx2asSearchArgs::new(\"8.0.0.0/8\")"); - println!(" .with_include_sub(true)"); - println!(" .with_limit(20);"); - println!(" let results = lens.search(&args)?;"); - println!(); - println!(" // Search with super-prefixes (less specific)"); - println!(" let args = Pfx2asSearchArgs::new(\"1.1.1.0/24\")"); - println!(" .with_include_super(true);"); - println!(" let results = lens.search(&args)?;"); - println!(" ```"); - - // Example 6: Query type detection - println!("\n6. Query type auto-detection:"); - println!(" The lens automatically detects query type:"); - println!(" - '13335' or 'AS13335' -> ASN query"); - println!(" - '1.1.1.0/24' -> Prefix query"); - println!(" - '2001:db8::/32' -> IPv6 prefix query"); - - // Example 7: Statistics - println!("\n7. Database statistics:"); - let record_count = pfx2as.record_count()?; - let prefix_count = pfx2as.prefix_count()?; - println!(" Total records: {}", record_count); - println!(" Unique prefixes: {}", prefix_count); - - let stats = pfx2as.validation_stats()?; - println!(" Validation stats:"); - println!(" - Valid: {}", stats.valid); - println!(" - Invalid: {}", stats.invalid); - println!(" - Unknown: {}", stats.unknown); - - // Example 8: Raw repository access - println!("\n8. Accessing raw connection for custom queries:"); - let conn = db.connection(); - - // Check what tables exist - let mut stmt = - conn.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'pfx2as%'")?; - let tables: Vec = stmt - .query_map([], |row| row.get(0))? - .filter_map(|r| r.ok()) - .collect(); - - println!(" Pfx2as tables in database:"); - for table in &tables { - println!(" - {}", table); - } - - // Example 9: Best practices - println!("\n9. Best practices:"); - println!(" - Check needs_refresh() before querying to ensure fresh data"); - println!(" - Use search() for most use cases (handles query type detection)"); - println!(" - Use with_limit() for large result sets"); - println!(" - RPKI validation requires RPKI data to be loaded (auto-loaded by lens)"); - println!(" - AS names require ASInfo data to be loaded"); - println!(" - For bulk operations, use the low-level repository methods"); - - // Example 10: Output formatting - println!("\n10. Output formatting:"); - println!(" The lens provides format_search_results() for various output formats:"); - println!(" - OutputFormat::Table: Pretty table with borders"); - println!(" - OutputFormat::Markdown: Markdown table"); - println!(" - OutputFormat::Json: Compact JSON"); - println!(" - OutputFormat::JsonPretty: Pretty-printed JSON"); - println!(" - OutputFormat::JsonLine: One JSON object per line"); - println!(" - OutputFormat::Psv: Pipe-separated values"); - - println!("\n=== Example Complete ==="); - Ok(()) -} diff --git a/examples/full/inspect_unified.rs b/examples/full/inspect_unified.rs deleted file mode 100644 index f614351..0000000 --- a/examples/full/inspect_unified.rs +++ /dev/null @@ -1,257 +0,0 @@ -//! Unified Inspection Example -//! -//! This example demonstrates using InspectLens for unified AS and prefix -//! information lookup, which aggregates data from multiple sources: -//! - ASInfo (core AS information, AS2Org, PeeringDB, hegemony, population) -//! - AS2Rel (AS-level relationships and connectivity) -//! - RPKI (ROAs and ASPAs) -//! - Pfx2as (prefix-to-ASN mappings) -//! -//! # Feature Requirements -//! -//! This example requires the `lens-full` feature, which includes all -//! lens functionality and dependencies. -//! -//! # Running -//! -//! ```bash -//! cargo run --example inspect_unified --features lens-full -//! ``` -//! -//! Note: This example requires network access to fetch data on first run. -//! It may take a minute to bootstrap all data sources. - -use monocle::database::MonocleDatabase; -use monocle::lens::inspect::{ - InspectDataSection, InspectDisplayConfig, InspectLens, InspectQueryOptions, -}; -use std::collections::HashSet; - -fn main() -> anyhow::Result<()> { - println!("=== Monocle Unified Inspection Example ===\n"); - - // Create an in-memory database for this example - // In production, use MonocleDatabase::open_in_dir("~/.monocle") - println!("Creating database..."); - let db = MonocleDatabase::open_in_memory()?; - let lens = InspectLens::new(&db); - - // Example 1: Check data availability - println!("\n1. Data availability:"); - println!(" ASInfo data available: {}", lens.is_data_available()); - println!(" Needs bootstrap: {}", lens.needs_bootstrap()); - println!(" Needs refresh: {}", lens.needs_refresh()); - - // Example 2: Ensure data is available (bootstrap/refresh as needed) - println!("\n2. Ensuring data availability..."); - println!(" (This may take a moment on first run)"); - match lens.ensure_data_available() { - Ok(summary) => { - println!(" Data refresh summary:"); - for msg in summary.format_messages() { - println!(" {}", msg); - } - if !summary.any_refreshed { - println!(" All data sources were up to date"); - } - } - Err(e) => { - println!(" Warning: Could not ensure data availability: {}", e); - println!(" (Continuing with available data...)"); - } - } - - // Example 3: Query type detection - println!("\n3. Query type detection:"); - let test_queries = [ - "13335", - "AS13335", - "as15169", - "1.1.1.0/24", - "8.8.8.0/24", - "2001:4860::/32", - "cloudflare", - "google", - ]; - - for query in &test_queries { - let query_type = lens.detect_query_type(query); - println!(" '{}' -> {:?}", query, query_type); - } - - // Example 4: Understanding query options - println!("\n4. InspectQueryOptions:"); - println!(" Available data sections:"); - for section in InspectDataSection::all() { - println!(" - {:?}", section); - } - - // Default options - let default_options = InspectQueryOptions::default(); - println!("\n Default options:"); - println!(" max_roas: {}", default_options.max_roas); - println!(" max_prefixes: {}", default_options.max_prefixes); - println!(" max_neighbors: {}", default_options.max_neighbors); - println!( - " max_search_results: {}", - default_options.max_search_results - ); - - // Full options (no limits) - let full_options = InspectQueryOptions::full(); - println!("\n Full options (no limits):"); - println!(" max_roas: {}", full_options.max_roas); - println!(" max_prefixes: {}", full_options.max_prefixes); - - // Example 5: Selective data sections - println!("\n5. Selecting specific data sections:"); - let mut select = HashSet::new(); - select.insert(InspectDataSection::Basic); - select.insert(InspectDataSection::Rpki); - - let selective_options = InspectQueryOptions { - select: Some(select), - ..Default::default() - }; - println!(" Selected sections: Basic, Rpki"); - println!(" Other sections will be omitted from results"); - - match lens.query_as_asn(&["13335".to_string()], &selective_options) { - Ok(result) => { - println!(" Query with selective options successful!"); - if let Some(q) = result.queries.first() { - println!(" Has Basic section: {}", q.asinfo.is_some()); - println!(" Has RPKI section: {}", q.rpki.is_some()); - println!(" Has Connectivity section: {}", q.connectivity.is_some()); - } - } - Err(e) => println!(" Query failed: {}", e), - } - - // Example 6: Query by ASN - println!("\n6. Query by ASN:"); - let options = InspectQueryOptions::default(); - match lens.query_as_asn(&["13335".to_string()], &options) { - Ok(result) => { - println!(" Query successful!"); - println!(" Number of query results: {}", result.queries.len()); - - // Format as JSON - let json = lens.format_json(&result, true); - println!(" JSON output (truncated):"); - for line in json.lines().take(20) { - println!(" {}", line); - } - if json.lines().count() > 20 { - println!(" ... (truncated)"); - } - } - Err(e) => { - println!(" Query failed: {}", e); - } - } - - // Example 7: Query by prefix - println!("\n7. Query by prefix:"); - match lens.query_as_prefix(&["1.1.1.0/24".to_string()], &options) { - Ok(result) => { - println!(" Query successful!"); - println!(" Number of query results: {}", result.queries.len()); - } - Err(e) => { - println!(" Query failed: {}", e); - } - } - - // Example 8: Query by name (search) - println!("\n8. Query by name:"); - match lens.query_as_name(&["cloudflare".to_string()], &options) { - Ok(result) => { - println!(" Query successful!"); - println!(" Number of query results: {}", result.queries.len()); - } - Err(e) => { - println!(" Query failed: {}", e); - } - } - - // Example 9: Auto-detect query type - println!("\n9. Auto-detect query type:"); - let queries = vec![ - "13335".to_string(), - "1.1.1.0/24".to_string(), - "cloudflare".to_string(), - ]; - match lens.query(&queries, &options) { - Ok(result) => { - println!(" Query successful!"); - println!( - " Processed {} queries with {} results", - queries.len(), - result.queries.len() - ); - } - Err(e) => { - println!(" Query failed: {}", e); - } - } - - // Example 10: Query by country - println!("\n10. Query by country:"); - match lens.query_by_country("US", &options) { - Ok(result) => { - println!(" Query for country 'US' successful!"); - println!(" Number of ASes found: {}", result.queries.len()); - } - Err(e) => { - println!(" Query failed: {}", e); - } - } - - // Example 11: Name lookup utilities - println!("\n11. Name lookup utilities:"); - if let Some(name) = lens.lookup_name(13335) { - println!(" AS13335 name: {}", name); - } else { - println!(" AS13335 name: not found"); - } - - if let Some(org) = lens.lookup_org(13335) { - println!(" AS13335 org: {}", org); - } else { - println!(" AS13335 org: not found"); - } - - // Example 12: Output formatting - println!("\n12. Output formatting:"); - println!(" Available formats:"); - println!(" - format_json(&result, pretty): JSON output"); - println!(" - format_table(&result, &config): Table output (requires display feature)"); - - let config = InspectDisplayConfig::auto(); - println!("\n Display config:"); - println!(" - terminal_width: {:?}", config.terminal_width); - println!(" - truncate_names: {}", config.truncate_names); - - // Example 13: Best practices - println!("\n13. Best practices:"); - println!(" - Call ensure_data_available() before queries"); - println!(" - Use selective options to reduce data transfer"); - println!(" - Cache InspectLens instance - it reuses database connection"); - println!(" - For bulk lookups, use query() with multiple items"); - println!(" - Use format_json() for API responses"); - - // Example 14: Data sources - println!("\n14. Data sources aggregated by InspectLens:"); - println!(" - ASInfo: Core AS information (name, country)"); - println!(" - AS2Org: Organization mapping from CAIDA"); - println!(" - PeeringDB: Network information"); - println!(" - Hegemony: IHR AS hegemony scores"); - println!(" - Population: APNIC user population estimates"); - println!(" - AS2Rel: AS-level relationships"); - println!(" - RPKI: ROAs and ASPAs"); - println!(" - Pfx2as: Prefix-to-ASN mappings"); - - println!("\n=== Example Complete ==="); - Ok(()) -} diff --git a/examples/full/progress_callbacks.rs b/examples/full/progress_callbacks.rs deleted file mode 100644 index 498049d..0000000 --- a/examples/full/progress_callbacks.rs +++ /dev/null @@ -1,429 +0,0 @@ -//! Progress Callbacks Example -//! -//! This example demonstrates how to use progress callbacks in monocle for -//! long-running operations like parsing and searching. Progress callbacks -//! are essential for building responsive GUI applications or showing -//! progress bars in CLI tools. -//! -//! # Feature Requirements -//! -//! This example requires the `lens-full` feature, which includes all -//! lens functionality. -//! -//! # Running -//! -//! ```bash -//! cargo run --example progress_callbacks --features lens-full -//! ``` - -use bgpkit_parser::BgpElem; -use monocle::lens::parse::{ParseFilters, ParseLens, ParseProgress}; -use monocle::lens::search::{SearchFilters, SearchLens, SearchProgress}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::Arc; -use std::time::Instant; - -fn main() -> anyhow::Result<()> { - println!("=== Monocle Progress Callbacks Example ===\n"); - - // ========================================================================== - // Part 1: ParseProgress Callbacks - // ========================================================================== - - println!("1. ParseProgress callback types:"); - println!(" ParseProgress::Started {{ file_path }}"); - println!(" - Emitted when parsing begins"); - println!(" - Contains the file path being parsed"); - println!(); - println!(" ParseProgress::Update {{ messages_processed, rate, elapsed_secs }}"); - println!(" - Emitted periodically during parsing (every 10,000 messages)"); - println!(" - messages_processed: Total count so far"); - println!(" - rate: Optional processing rate (messages/sec)"); - println!(" - elapsed_secs: Time since start"); - println!(); - println!(" ParseProgress::Completed {{ total_messages, duration_secs, rate }}"); - println!(" - Emitted when parsing finishes"); - println!(" - Contains final statistics"); - - // ========================================================================== - // Part 2: Creating a Parse Progress Callback - // ========================================================================== - - println!("\n2. Creating a parse progress callback:"); - - // Shared state for the callback - let parse_started = Arc::new(AtomicBool::new(false)); - let parse_messages = Arc::new(AtomicU64::new(0)); - let parse_start_time = Arc::new(std::sync::Mutex::new(None::)); - - // Clone for use in callback - let started_clone = parse_started.clone(); - let messages_clone = parse_messages.clone(); - let start_time_clone = parse_start_time.clone(); - - let parse_callback = Arc::new(move |progress: ParseProgress| match progress { - ParseProgress::Started { file_path } => { - started_clone.store(true, Ordering::SeqCst); - *start_time_clone.lock().unwrap() = Some(Instant::now()); - println!(" [PARSE] Started: {}", file_path); - } - ParseProgress::Update { - messages_processed, - rate, - elapsed_secs, - } => { - messages_clone.store(messages_processed, Ordering::SeqCst); - let rate_str = rate - .map(|r| format!("{:.0} msg/s", r)) - .unwrap_or_else(|| "N/A".to_string()); - println!( - " [PARSE] Progress: {} messages, {}, {:.1}s", - messages_processed, rate_str, elapsed_secs - ); - } - ParseProgress::Completed { - total_messages, - duration_secs, - rate, - } => { - messages_clone.store(total_messages, Ordering::SeqCst); - let rate_str = rate - .map(|r| format!("{:.0} msg/s", r)) - .unwrap_or_else(|| "N/A".to_string()); - println!( - " [PARSE] Completed: {} messages in {:.2}s ({})", - total_messages, duration_secs, rate_str - ); - } - }); - - println!(" Parse callback created with shared state tracking"); - - // ========================================================================== - // Part 3: Using Parse Callback (demonstration) - // ========================================================================== - - println!("\n3. Using parse callback (executing):"); - - let lens = ParseLens::new(); - let filters = ParseFilters { - origin_asn: vec!["13335".to_string()], - ..Default::default() - }; - - // Use a real URL (Cloudflare RRC00 update file) - let url = "https://data.ris.ripe.net/rrc00/2024.01/updates.20240101.0000.gz"; - - println!(" Parsing {} ...", url); - let elems = lens.parse_with_progress(&filters, url, Some(parse_callback))?; - - println!(" Found {} elements matching filter", elems.len()); - - // ========================================================================== - // Part 4: SearchProgress Callbacks - // ========================================================================== - - println!("\n4. SearchProgress callback types:"); - println!(" SearchProgress::QueryingBroker"); - println!(" - Emitted when starting broker query"); - println!(); - println!(" SearchProgress::FilesFound {{ count }}"); - println!(" - Emitted after broker query completes"); - println!(" - count: Number of MRT files to process"); - println!(); - println!(" SearchProgress::FileStarted {{ file_index, total_files, file_url, collector }}"); - println!(" - Emitted when starting to process a file"); - println!(); - println!(" SearchProgress::FileCompleted {{ file_index, total_files, messages_found, success, error }}"); - println!(" - Emitted when a file finishes processing"); - println!(); - println!(" SearchProgress::ProgressUpdate {{ files_completed, total_files, total_messages, percent_complete, elapsed_secs, eta_secs }}"); - println!(" - Periodic overall progress update"); - println!(); - println!(" SearchProgress::Completed {{ total_files, successful_files, failed_files, total_messages, duration_secs, files_per_sec }}"); - println!(" - Final summary when search completes"); - - // ========================================================================== - // Part 5: Creating a Search Progress Callback - // ========================================================================== - - println!("\n5. Creating a search progress callback:"); - - // Shared state for search callback - let search_files_total = Arc::new(AtomicU64::new(0)); - let search_files_done = Arc::new(AtomicU64::new(0)); - let search_messages = Arc::new(AtomicU64::new(0)); - - let files_total_clone = search_files_total.clone(); - let files_done_clone = search_files_done.clone(); - let messages_clone2 = search_messages.clone(); - - let search_callback = Arc::new(move |progress: SearchProgress| match progress { - SearchProgress::QueryingBroker => { - println!(" [SEARCH] Querying BGPKIT broker..."); - } - SearchProgress::FilesFound { count } => { - files_total_clone.store(count as u64, Ordering::SeqCst); - println!(" [SEARCH] Found {} files to process", count); - } - SearchProgress::FileStarted { - file_index, - total_files, - collector, - .. - } => { - println!( - " [SEARCH] [{}/{}] Starting {} ...", - file_index + 1, - total_files, - collector - ); - } - SearchProgress::FileCompleted { - file_index, - total_files, - messages_found, - success, - error, - } => { - files_done_clone.fetch_add(1, Ordering::SeqCst); - if success { - println!( - " [SEARCH] [{}/{}] Completed: {} messages", - file_index + 1, - total_files, - messages_found - ); - } else { - println!( - " [SEARCH] [{}/{}] Failed: {}", - file_index + 1, - total_files, - error.unwrap_or_else(|| "Unknown error".to_string()) - ); - } - } - SearchProgress::ProgressUpdate { - files_completed, - total_files, - total_messages, - percent_complete, - elapsed_secs, - eta_secs, - } => { - messages_clone2.store(total_messages, Ordering::SeqCst); - let eta_str = eta_secs - .map(|e| format!("ETA: {:.0}s", e)) - .unwrap_or_else(|| "ETA: N/A".to_string()); - println!( - " [SEARCH] Progress: {}/{} files ({:.1}%), {} messages, {:.1}s elapsed, {}", - files_completed, - total_files, - percent_complete, - total_messages, - elapsed_secs, - eta_str - ); - } - SearchProgress::Completed { - total_files, - successful_files, - failed_files, - total_messages, - duration_secs, - files_per_sec, - } => { - let rate_str = files_per_sec - .map(|r| format!("{:.2} files/s", r)) - .unwrap_or_else(|| "N/A".to_string()); - println!( - " [SEARCH] Completed: {}/{} files successful, {} failed", - successful_files, total_files, failed_files - ); - println!( - " [SEARCH] Total: {} messages in {:.2}s ({})", - total_messages, duration_secs, rate_str - ); - } - }); - - println!(" Search callback created with shared state tracking"); - - // ========================================================================== - // Part 6: Element Handler - // ========================================================================== - - println!("\n6. Element handler for streaming results:"); - println!(" The element handler is called for each matching BGP element."); - println!(" It must be Send + Sync for thread safety (parallel processing)."); - - let element_count = Arc::new(AtomicU64::new(0)); - let count_clone = element_count.clone(); - - let element_handler = Arc::new(move |_elem: BgpElem, collector: String| { - let count = count_clone.fetch_add(1, Ordering::SeqCst); - if count % 100 == 0 { - println!(" [ELEM] Received element {} from {}", count, collector); - } - }); - - println!(" Element handler created"); - - println!("\n Executing search (Cloudflare AS13335, 10 minutes)..."); - let search_lens = SearchLens::new(); - let search_filters = SearchFilters { - parse_filters: ParseFilters { - origin_asn: vec!["13335".to_string()], - start_ts: Some("2024-01-01T00:00:00Z".to_string()), - end_ts: Some("2024-01-01T00:10:00Z".to_string()), - ..Default::default() - }, - ..Default::default() - }; - - search_lens.search_with_progress(&search_filters, Some(search_callback), element_handler)?; - - // ========================================================================== - // Part 7: GUI Integration Pattern - // ========================================================================== - - println!("\n7. GUI integration pattern:"); - println!(" For GUI applications, callbacks can send messages to the UI thread:"); - println!(); - println!(" ```rust"); - println!(" use std::sync::mpsc;"); - println!(); - println!(" // Create a channel for UI updates"); - println!(" let (tx, rx) = mpsc::channel();"); - println!(); - println!(" // Clone sender for callback"); - println!(" let tx_clone = tx.clone();"); - println!(" let callback = Arc::new(move |progress: SearchProgress| {{"); - println!(" // Send progress to UI thread (non-blocking)"); - println!(" let _ = tx_clone.send(UiMessage::SearchProgress(progress));"); - println!(" }});"); - println!(); - println!(" // In UI thread, receive and handle messages"); - println!(" while let Ok(msg) = rx.try_recv() {{"); - println!(" match msg {{"); - println!(" UiMessage::SearchProgress(p) => update_progress_bar(p),"); - println!(" // ...other messages"); - println!(" }}"); - println!(" }}"); - println!(" ```"); - - // ========================================================================== - // Part 8: Async Integration Pattern - // ========================================================================== - - println!("\n8. Async integration pattern:"); - println!(" For async applications, use tokio channels:"); - println!(); - println!(" ```rust"); - println!(" use tokio::sync::mpsc;"); - println!(); - println!(" let (tx, mut rx) = mpsc::unbounded_channel();"); - println!(); - println!(" let callback = Arc::new(move |progress: SearchProgress| {{"); - println!(" let _ = tx.send(progress);"); - println!(" }});"); - println!(); - println!(" // Spawn search on blocking thread"); - println!(" let handle = tokio::task::spawn_blocking(move || {{"); - println!(" lens.search_with_progress(&filters, Some(callback), handler)"); - println!(" }});"); - println!(); - println!(" // Process progress in async context"); - println!(" while let Some(progress) = rx.recv().await {{"); - println!(" // Update UI, log, etc."); - println!(" }}"); - println!(" ```"); - - // ========================================================================== - // Part 9: Progress Bar Integration - // ========================================================================== - - println!("\n9. Progress bar integration (indicatif):"); - println!(" ```rust"); - println!(" use indicatif::{{ProgressBar, ProgressStyle}};"); - println!(); - println!(" let pb = ProgressBar::new(100);"); - println!(" pb.set_style(ProgressStyle::default_bar()"); - println!(" .template(\"[{{bar:40}}] {{pos}}/{{len}} {{msg}}\"));"); - println!(); - println!(" let pb_clone = pb.clone();"); - println!(" let callback = Arc::new(move |progress: SearchProgress| {{"); - println!(" match progress {{"); - println!(" SearchProgress::FilesFound {{ count }} => {{"); - println!(" pb_clone.set_length(count as u64);"); - println!(" }}"); - println!(" SearchProgress::FileCompleted {{ file_index, .. }} => {{"); - println!(" pb_clone.set_position(file_index as u64 + 1);"); - println!(" }}"); - println!(" SearchProgress::Completed {{ .. }} => {{"); - println!(" pb_clone.finish_with_message(\"done\");"); - println!(" }}"); - println!(" _ => {{}}"); - println!(" }}"); - println!(" }});"); - println!(" ```"); - - // ========================================================================== - // Part 10: Serialization for IPC - // ========================================================================== - - println!("\n10. Serialization for IPC:"); - println!(" Both ParseProgress and SearchProgress implement Serialize/Deserialize."); - println!(" This allows sending progress over WebSocket, IPC, etc."); - println!(); - - // Demonstrate serialization - let progress = SearchProgress::FilesFound { count: 42 }; - let json = serde_json::to_string(&progress)?; - println!(" Example serialized progress:"); - println!(" {}", json); - - let progress = SearchProgress::ProgressUpdate { - files_completed: 10, - total_files: 42, - total_messages: 50000, - percent_complete: 23.8, - elapsed_secs: 30.5, - eta_secs: Some(97.5), - }; - let json = serde_json::to_string_pretty(&progress)?; - println!("\n Complex progress:"); - println!("{}", json); - - // ========================================================================== - // Part 11: Best Practices - // ========================================================================== - - println!("\n11. Best practices:"); - println!(" - Keep callbacks lightweight (don't block)"); - println!(" - Use channels for cross-thread communication"); - println!(" - Handle all progress variants (use _ => {{}} for unhandled)"); - println!(" - Track state with Arc for thread safety"); - println!(" - Consider rate-limiting UI updates for performance"); - println!(" - Always handle the Completed variant for cleanup"); - println!(" - Check 'success' field in FileCompleted for error handling"); - - // ========================================================================== - // Part 12: Error Handling in Callbacks - // ========================================================================== - - println!("\n12. Error handling in callbacks:"); - println!(" Callbacks should not panic - this could crash parallel workers."); - println!(" Use Result types and logging for error handling:"); - println!(); - println!(" ```rust"); - println!(" let callback = Arc::new(move |progress: SearchProgress| {{"); - println!(" if let Err(e) = handle_progress(&progress) {{"); - println!(" tracing::warn!(\"Progress handler error: {{}}\", e);"); - println!(" }}"); - println!(" }});"); - println!(" ```"); - - println!("\n=== Example Complete ==="); - Ok(()) -} diff --git a/examples/inspect_lens.rs b/examples/inspect_lens.rs new file mode 100644 index 0000000..78aa399 --- /dev/null +++ b/examples/inspect_lens.rs @@ -0,0 +1,50 @@ +//! Inspect Example +//! +//! Demonstrates unified AS and prefix information lookup. +//! +//! # Running +//! +//! ```bash +//! cargo run --example inspect --features lib +//! ``` +//! +//! Note: Requires network access to fetch data on first run. + +use monocle::config::MonocleConfig; +use monocle::database::MonocleDatabase; +use monocle::lens::inspect::{InspectLens, InspectQueryOptions}; + +fn main() -> anyhow::Result<()> { + let db = MonocleDatabase::open_in_memory()?; + let config = MonocleConfig::default(); + let lens = InspectLens::new(&db, &config); + + // Ensure data is available + println!("Loading data..."); + lens.ensure_data_available()?; + + // Query by ASN + println!("\nQuerying AS13335:"); + let result = lens.query_as_asn(&["13335".to_string()], &InspectQueryOptions::default())?; + if let Some(q) = result.queries.first() { + if let Some(ref info) = q.asinfo { + if let Some(ref detail) = info.detail { + println!(" Name: {}", detail.core.name); + } + } + } + + // Query by prefix + println!("\nQuerying 1.1.1.0/24:"); + let result = + lens.query_as_prefix(&["1.1.1.0/24".to_string()], &InspectQueryOptions::default())?; + if let Some(q) = result.queries.first() { + if let Some(ref pfx) = q.prefix { + if let Some(ref info) = pfx.pfx2as { + println!(" Origin AS: {:?}", info.origin_asns); + } + } + } + + Ok(()) +} diff --git a/examples/ip_lens.rs b/examples/ip_lens.rs new file mode 100644 index 0000000..69f8b79 --- /dev/null +++ b/examples/ip_lens.rs @@ -0,0 +1,39 @@ +//! IP Information Example +//! +//! Demonstrates looking up IP address information including ASN and RPKI status. +//! +//! # Running +//! +//! ```bash +//! cargo run --example ip_lens --features lib +//! ``` + +use monocle::lens::ip::{IpLens, IpLookupArgs}; +use std::net::IpAddr; + +fn main() -> anyhow::Result<()> { + let lens = IpLens::new(); + + // Look up a specific IP + let ip = "1.1.1.1".parse::()?; + let args = IpLookupArgs::new(ip); + let info = lens.lookup(&args)?; + + println!("IP: {}", info.ip); + if let Some(country) = &info.country { + println!("Location: {}", country); + } + + if let Some(asn) = &info.asn { + println!("\nNetwork Information:"); + println!(" ASN: {}", asn.asn); + println!(" Name: {}", asn.name); + println!(" Prefix: {}", asn.prefix); + println!(" RPKI: {}", asn.rpki); + if let Some(country) = &asn.country { + println!(" Country: {}", country); + } + } + + Ok(()) +} diff --git a/examples/parse_lens.rs b/examples/parse_lens.rs new file mode 100644 index 0000000..349cb3d --- /dev/null +++ b/examples/parse_lens.rs @@ -0,0 +1,37 @@ +//! MRT Parsing Example +//! +//! Demonstrates parsing MRT files with filters. +//! +//! # Running +//! +//! ```bash +//! cargo run --example mrt_parsing --features lib +//! ``` + +use monocle::lens::parse::{ParseFilters, ParseLens}; + +fn main() -> anyhow::Result<()> { + let lens = ParseLens::new(); + + // Parse with filters + let filters = ParseFilters { + origin_asn: vec!["13335".to_string()], + ..Default::default() + }; + + println!("Parsing MRT file with filters:"); + println!(" Origin ASN: 13335 (Cloudflare)"); + + let url = "https://data.ris.ripe.net/rrc00/2024.01/updates.20240101.0000.gz"; + let elems = lens.parse_with_progress(&filters, url, None)?; + + println!("\nFound {} BGP elements", elems.len()); + for elem in elems.iter().take(3) { + println!( + " {} - {} - {:?}", + elem.timestamp, elem.prefix, elem.elem_type + ); + } + + Ok(()) +} diff --git a/examples/pfx2as_lens.rs b/examples/pfx2as_lens.rs new file mode 100644 index 0000000..de8dfc7 --- /dev/null +++ b/examples/pfx2as_lens.rs @@ -0,0 +1,45 @@ +//! Prefix-to-ASN Example +//! +//! Demonstrates prefix-to-ASN mapping lookups with RPKI validation. +//! +//! # Running +//! +//! ```bash +//! cargo run --example pfx2as_lens --features lib +//! ``` + +use monocle::database::MonocleDatabase; +use monocle::lens::pfx2as::{Pfx2asLens, Pfx2asSearchArgs}; +use std::time::Duration; + +fn main() -> anyhow::Result<()> { + let db = MonocleDatabase::open_in_memory()?; + let lens = Pfx2asLens::new(&db); + + // Refresh cache if needed + let ttl = Duration::from_secs(24 * 60 * 60); + if lens.needs_refresh(ttl)? { + println!("Refreshing pfx2as cache..."); + lens.refresh(None)?; + } + + // Search by prefix + println!("\nSearching for 1.1.1.0/24:"); + let args = Pfx2asSearchArgs::new("1.1.1.0/24").with_show_name(true); + let results = lens.search(&args)?; + + for r in &results { + println!(" {} -> AS{} (RPKI: {})", r.prefix, r.origin_asn, r.rpki); + } + + // Search by ASN + println!("\nSearching for AS13335 prefixes:"); + let args = Pfx2asSearchArgs::new("13335").with_limit(5); + let results = lens.search(&args)?; + + for r in &results { + println!(" {} -> AS{}", r.prefix, r.origin_asn); + } + + Ok(()) +} diff --git a/examples/rpki_lens.rs b/examples/rpki_lens.rs new file mode 100644 index 0000000..2f6a666 --- /dev/null +++ b/examples/rpki_lens.rs @@ -0,0 +1,54 @@ +//! RPKI Validation Example +//! +//! Demonstrates RPKI validation for prefix-ASN pairs. +//! +//! # Running +//! +//! ```bash +//! cargo run --example rpki_validation --features lib +//! ``` +//! +//! Note: Requires network access to fetch RPKI data on first run. + +use monocle::database::MonocleDatabase; +use monocle::lens::rpki::{RpkiLens, RpkiValidationState}; +use std::time::Duration; + +fn main() -> anyhow::Result<()> { + let db = MonocleDatabase::open_in_memory()?; + let lens = RpkiLens::new(&db); + + // Refresh cache if needed + let ttl = Duration::from_secs(24 * 60 * 60); + if lens.needs_refresh(ttl)? { + println!("Refreshing RPKI cache..."); + lens.refresh()?; + } + + // Validate prefix-ASN pairs + let tests = [ + ("1.1.1.0/24", 13335, "Cloudflare"), + ("8.8.8.0/24", 15169, "Google"), + ("1.1.1.0/24", 12345, "Invalid ASN"), + ]; + + println!("\nRPKI Validation Results:"); + for (prefix, asn, desc) in &tests { + match lens.validate(prefix, *asn) { + Ok(result) => { + let icon = match result.state { + RpkiValidationState::Valid => "✓", + RpkiValidationState::Invalid => "✗", + RpkiValidationState::NotFound => "?", + }; + println!( + " {} {} AS{} - {} ({})", + icon, prefix, asn, result.state, desc + ); + } + Err(e) => println!(" ! Error: {}", e), + } + } + + Ok(()) +} diff --git a/examples/search_lens.rs b/examples/search_lens.rs new file mode 100644 index 0000000..9aca18f --- /dev/null +++ b/examples/search_lens.rs @@ -0,0 +1,52 @@ +//! Search BGP Messages Example +//! +//! Demonstrates searching for BGP messages using the broker. +//! +//! # Running +//! +//! ```bash +//! cargo run --example search_bgp_messages --features lib +//! ``` + +use monocle::lens::parse::ParseFilters; +use monocle::lens::search::{SearchDumpType, SearchFilters}; + +fn main() -> anyhow::Result<()> { + // Search for BGP updates from first hour of 2025 + let filters = SearchFilters { + parse_filters: ParseFilters { + start_ts: Some("2025-01-01T00:00:00Z".to_string()), + end_ts: Some("2025-01-01T01:00:00Z".to_string()), + ..Default::default() + }, + collector: Some("rrc00".to_string()), + project: Some("riperis".to_string()), + dump_type: SearchDumpType::Updates, + }; + + println!("Searching for BGP messages..."); + + let broker = filters.build_broker()?; + let items = broker.query()?; + + println!("Found {} MRT files", items.len()); + + if let Some(first) = items.first() { + println!("\nProcessing: {}", first.url); + + let parser = filters.to_parser(&first.url)?; + + let mut count = 0; + for elem in parser.into_iter().take(5) { + count += 1; + println!( + " {} - {} - {:?}", + elem.timestamp, elem.prefix, elem.elem_type + ); + } + + println!("\nShowing first {} elements", count); + } + + Ok(()) +} diff --git a/examples/standalone/output_formats.rs b/examples/standalone/output_formats.rs deleted file mode 100644 index 0629db6..0000000 --- a/examples/standalone/output_formats.rs +++ /dev/null @@ -1,166 +0,0 @@ -//! Output Formats Example -//! -//! This example demonstrates the unified OutputFormat type and how to work -//! with different output formats in monocle. -//! -//! # Feature Requirements -//! -//! This example only requires the `lens-core` feature, which has minimal -//! dependencies. -//! -//! # Running -//! -//! ```bash -//! cargo run --example output_formats --features lens-core -//! ``` - -use monocle::lens::utils::OutputFormat; -use serde::{Deserialize, Serialize}; -use std::str::FromStr; - -/// Example data structure that can be formatted in different ways -#[derive(Debug, Clone, Serialize, Deserialize)] -struct ExampleRecord { - id: u32, - name: String, - value: f64, - active: bool, -} - -fn main() -> anyhow::Result<()> { - println!("=== Monocle Output Formats Example ===\n"); - - // Example 1: List all available format names - println!("1. Available output formats:"); - for name in OutputFormat::all_names() { - println!(" - {}", name); - } - - // Example 2: Parse format from string - println!("\n2. Parsing format names:"); - let format_strings = [ - "table", - "markdown", - "md", - "json", - "json-pretty", - "jsonpretty", - "json-line", - "jsonl", - "ndjson", - "psv", - "pipe", - ]; - - for s in &format_strings { - match OutputFormat::from_str(s) { - Ok(fmt) => println!(" '{}' -> {:?}", s, fmt), - Err(e) => println!(" '{}' -> Error: {}", s, e), - } - } - - // Example 3: Check format type - println!("\n3. Format type checking:"); - let formats = [ - OutputFormat::Table, - OutputFormat::Markdown, - OutputFormat::Json, - OutputFormat::JsonPretty, - OutputFormat::JsonLine, - OutputFormat::Psv, - ]; - - for fmt in &formats { - println!( - " {:?}: is_json={}, is_table={}", - fmt, - fmt.is_json(), - fmt.is_table() - ); - } - - // Example 4: Display format names - println!("\n4. Format display names:"); - for fmt in &formats { - println!(" {:?} displays as '{}'", fmt, fmt); - } - - // Example 5: Using formats with data - println!("\n5. Formatting example data:"); - - let records = vec![ - ExampleRecord { - id: 1, - name: "Cloudflare".to_string(), - value: 99.9, - active: true, - }, - ExampleRecord { - id: 2, - name: "Google".to_string(), - value: 98.5, - active: true, - }, - ExampleRecord { - id: 3, - name: "Example".to_string(), - value: 50.0, - active: false, - }, - ]; - - // JSON format - println!(" JSON format:"); - let json = serde_json::to_string(&records)?; - println!(" {}", json); - - // JSON Pretty format - println!("\n JSON Pretty format:"); - let json_pretty = serde_json::to_string_pretty(&records)?; - println!("{}", json_pretty); - - // JSON Lines format - println!(" JSON Lines format:"); - for record in &records { - println!(" {}", serde_json::to_string(record)?); - } - - // PSV (Pipe-Separated Values) format - println!("\n PSV format:"); - println!(" id|name|value|active"); - for record in &records { - println!( - " {}|{}|{}|{}", - record.id, record.name, record.value, record.active - ); - } - - // Example 6: Default format - println!("\n6. Default format:"); - let default_fmt = OutputFormat::default(); - println!(" Default format is: {:?}", default_fmt); - - // Example 7: Pattern matching on formats - println!("\n7. Pattern matching for format-specific logic:"); - for fmt in &formats { - let description = match fmt { - OutputFormat::Table => "Pretty table with borders - great for terminal output", - OutputFormat::Markdown => "Markdown table - great for documentation", - OutputFormat::Json => "Compact JSON - great for piping to jq", - OutputFormat::JsonPretty => "Pretty JSON - great for human reading", - OutputFormat::JsonLine => "JSON Lines - great for streaming/logs", - OutputFormat::Psv => "Pipe-separated - great for simple parsing", - }; - println!(" {:?}: {}", fmt, description); - } - - // Example 8: Error handling for invalid formats - println!("\n8. Error handling for invalid format:"); - match OutputFormat::from_str("invalid_format") { - Ok(_) => println!(" Unexpectedly succeeded"), - Err(e) => println!(" Error (expected): {}", e), - } - - println!("\n=== Example Complete ==="); - Ok(()) -} diff --git a/examples/standalone/time_parsing.rs b/examples/standalone/time_parsing.rs deleted file mode 100644 index 16a51de..0000000 --- a/examples/standalone/time_parsing.rs +++ /dev/null @@ -1,120 +0,0 @@ -//! Time Parsing Example -//! -//! This example demonstrates using TimeLens for time parsing and conversion. -//! -//! # Feature Requirements -//! -//! This example only requires the `lens-core` feature, which has minimal -//! dependencies (chrono, chrono-humanize, dateparser). -//! -//! # Running -//! -//! ```bash -//! cargo run --example time_parsing --features lens-core -//! ``` - -use monocle::lens::time::{TimeBgpTime, TimeLens, TimeOutputFormat, TimeParseArgs}; - -fn main() -> anyhow::Result<()> { - println!("=== Monocle Time Parsing Example ===\n"); - - let lens = TimeLens::new(); - - // Example 1: Parse current time - println!("1. Current time:"); - let args = TimeParseArgs::now(); - let results = lens.parse(&args)?; - print_results(&results); - - // Example 2: Parse Unix timestamp - println!("\n2. Parse Unix timestamp (1697043600):"); - let args = TimeParseArgs::new(vec!["1697043600".to_string()]); - let results = lens.parse(&args)?; - print_results(&results); - - // Example 3: Parse RFC3339 string - println!("\n3. Parse RFC3339 string:"); - let args = TimeParseArgs::new(vec!["2023-10-11T15:00:00Z".to_string()]); - let results = lens.parse(&args)?; - print_results(&results); - - // Example 4: Parse human-readable date - println!("\n4. Parse human-readable date:"); - let args = TimeParseArgs::new(vec!["October 11, 2023".to_string()]); - let results = lens.parse(&args)?; - print_results(&results); - - // Example 5: Parse multiple times at once - println!("\n5. Parse multiple times:"); - let args = TimeParseArgs::new(vec![ - "1697043600".to_string(), - "2024-01-01T00:00:00Z".to_string(), - "January 15, 2024".to_string(), - ]); - let results = lens.parse(&args)?; - print_results(&results); - - // Example 6: Using different output formats - println!("\n6. Different output formats:"); - - let args = TimeParseArgs::new(vec!["2024-06-15T12:30:00Z".to_string()]); - let results = lens.parse(&args)?; - - println!(" RFC3339 format:"); - println!( - " {}", - lens.format_results(&results, &TimeOutputFormat::Rfc3339) - ); - - println!(" Unix timestamp format:"); - println!( - " {}", - lens.format_results(&results, &TimeOutputFormat::Unix) - ); - - println!(" JSON format:"); - println!("{}", lens.format_results(&results, &TimeOutputFormat::Json)); - - // Example 7: Direct time string parsing - println!("\n7. Direct time string parsing:"); - let dt = lens.parse_time_string("2024-03-14T09:26:53Z")?; - println!(" Parsed DateTime: {}", dt); - println!(" Unix timestamp: {}", dt.timestamp()); - - // Example 8: Convert to RFC3339 strings - println!("\n8. Batch convert to RFC3339:"); - let times = vec![ - "1700000000".to_string(), - "1710000000".to_string(), - "1720000000".to_string(), - ]; - let rfc3339_strings = lens.parse_to_rfc3339(×)?; - for (orig, converted) in times.iter().zip(rfc3339_strings.iter()) { - println!(" {} -> {}", orig, converted); - } - - // Example 9: Using format_json for API responses - println!("\n9. JSON output for API integration:"); - let args = TimeParseArgs::new(vec!["2024-06-15T12:30:00Z".to_string()]); - let results = lens.parse(&args)?; - - println!(" Compact JSON:"); - println!(" {}", lens.format_json(&results, false)); - - println!(" Pretty JSON:"); - println!("{}", lens.format_json(&results, true)); - - println!("\n=== Example Complete ==="); - Ok(()) -} - -fn print_results(results: &[TimeBgpTime]) { - for t in results { - println!(" Unix: {}", t.unix); - println!(" RFC3339: {}", t.rfc3339); - println!(" Human: {}", t.human); - if results.len() > 1 { - println!(" ---"); - } - } -} diff --git a/examples/time_lens.rs b/examples/time_lens.rs new file mode 100644 index 0000000..605d700 --- /dev/null +++ b/examples/time_lens.rs @@ -0,0 +1,37 @@ +//! Time Parsing Example +//! +//! Demonstrates parsing timestamps from various formats and converting between them. +//! +//! # Running +//! +//! ```bash +//! cargo run --example time_parsing --features lib +//! ``` + +use monocle::lens::time::{TimeLens, TimeOutputFormat, TimeParseArgs}; + +fn main() -> anyhow::Result<()> { + let lens = TimeLens::new(); + + // Parse various time formats + let args = TimeParseArgs::new(vec![ + "1697043600".to_string(), // Unix timestamp + "2023-10-11T15:00:00Z".to_string(), // RFC3339 + "October 11, 2023".to_string(), // Human-readable + ]); + + let results = lens.parse(&args)?; + + // Display results in different formats + println!("Parsed Times:"); + println!( + "{}", + lens.format_results(&results, &TimeOutputFormat::Table) + ); + + // Convert to JSON for API usage + println!("\nJSON Output:"); + println!("{}", lens.format_json(&results, false)); + + Ok(()) +} diff --git a/src/bin/commands/as2rel.rs b/src/bin/commands/as2rel.rs index 3149080..6a5e19c 100644 --- a/src/bin/commands/as2rel.rs +++ b/src/bin/commands/as2rel.rs @@ -5,6 +5,7 @@ use monocle::lens::utils::{truncate_name, OutputFormat, DEFAULT_NAME_MAX_LEN}; use monocle::MonocleConfig; use serde::Serialize; use serde_json::json; +use std::time::Duration; use tabled::settings::Style; use tabled::Table; @@ -79,12 +80,7 @@ pub struct As2relArgs { pub is_peer: bool, } -pub fn run( - config: &MonocleConfig, - args: As2relArgs, - output_format: OutputFormat, - no_refresh: bool, -) { +pub fn run(config: &MonocleConfig, args: As2relArgs, output_format: OutputFormat, no_update: bool) { let As2relArgs { asns, update, @@ -135,8 +131,8 @@ pub fn run( // Handle explicit updates if update || update_with.is_some() { - if no_refresh { - eprintln!("[monocle] Warning: --update ignored because --no-refresh is set"); + if no_update { + eprintln!("[monocle] Warning: --update ignored because --no-update is set"); } else { eprintln!("[monocle] Updating AS2rel data..."); @@ -148,7 +144,7 @@ pub fn run( } }; - let lens = As2relLens::new(&db); + let lens = As2relLens::with_ttl(&db, config.as2rel_cache_ttl()); let result = match &update_with { Some(path) => lens.update_from(path), None => lens.update(), @@ -181,6 +177,7 @@ pub fn run( is_upstream, is_downstream, is_peer, + config.as2rel_cache_ttl(), ); return; } @@ -195,16 +192,16 @@ pub fn run( } }; - let lens = As2relLens::new(&db); + let lens = As2relLens::with_ttl(&db, config.as2rel_cache_ttl()); // Check if data needs to be initialized or updated automatically if let Some(reason) = lens.update_reason() { - if no_refresh { + if no_update { eprintln!( "[monocle] Warning: AS2rel {} Results may be incomplete.", reason ); - eprintln!("[monocle] Run without --no-refresh or use 'monocle config db-refresh --as2rel' to load data."); + eprintln!("[monocle] Run without --no-update or use 'monocle config update --as2rel' to load data."); } else { eprintln!("[monocle] AS2rel {}, updating now...", reason); @@ -237,6 +234,7 @@ pub fn run( is_upstream, is_downstream, is_peer, + config.as2rel_cache_ttl(), ); } @@ -275,8 +273,9 @@ fn run_query( is_upstream: bool, is_downstream: bool, is_peer: bool, + ttl: Duration, ) { - let lens = As2relLens::new(db); + let lens = As2relLens::with_ttl(db, ttl); // Build search args let search_args = As2relSearchArgs { diff --git a/src/bin/commands/config.rs b/src/bin/commands/config.rs index e84f27b..8d17dbe 100644 --- a/src/bin/commands/config.rs +++ b/src/bin/commands/config.rs @@ -1,3 +1,4 @@ +use chrono_humanize::HumanTime; use clap::{Args, Subcommand}; use monocle::config::{ format_size, get_data_source_info, get_sqlite_info, DataSource, DataSourceStatus, @@ -12,6 +13,19 @@ use serde::Serialize; use std::path::Path; use std::time::Instant; +/// Convert a timestamp string like "2024-01-15 10:30:00 UTC" to relative time like "2 hours ago" +fn to_relative_time(timestamp_str: &str) -> String { + // Parse the timestamp string format: "YYYY-MM-DD HH:MM:SS UTC" + if let Ok(naive) = chrono::NaiveDateTime::parse_from_str(timestamp_str, "%Y-%m-%d %H:%M:%S UTC") + { + let dt = naive.and_utc(); + HumanTime::from(dt).to_string() + } else { + // If parsing fails, return the original string + timestamp_str.to_string() + } +} + /// Arguments for the Config command #[derive(Args)] pub struct ConfigArgs { @@ -27,52 +41,63 @@ pub struct ConfigArgs { #[derive(Subcommand)] #[allow(clippy::enum_variant_names)] pub enum ConfigCommands { - /// Refresh data source(s) - DbRefresh { - /// Refresh asinfo data + /// Update data source(s) + Update { + /// Update asinfo data #[clap(long)] asinfo: bool, - /// Refresh as2rel data + /// Update as2rel data #[clap(long)] as2rel: bool, - /// Refresh RPKI data + /// Update RPKI data #[clap(long)] rpki: bool, - /// Refresh pfx2as data + /// Update pfx2as data #[clap(long)] pfx2as: bool, /// RTR endpoint for fetching ROAs (format: host:port) - /// Overrides config file setting for this refresh only. + /// Overrides config file setting for this update only. /// Example: --rtr-endpoint rtr.rpki.cloudflare.com:8282 #[clap(long, value_name = "HOST:PORT")] rtr_endpoint: Option, }, /// Backup the database to a destination - DbBackup { + Backup { /// Destination path for the backup #[clap(value_name = "DEST")] destination: String, }, /// List available data sources and their status - DbSources, + Sources, } #[derive(Debug, Serialize)] struct ConfigInfo { config_file: String, data_dir: String, + cache_ttl: CacheTtlConfig, database: SqliteDatabaseInfo, server_defaults: ServerDefaults, #[serde(skip_serializing_if = "Option::is_none")] + rtr_endpoint: Option, + #[serde(skip_serializing_if = "Option::is_none")] files: Option>, } +#[derive(Debug, Serialize)] +struct CacheTtlConfig { + asinfo_secs: u64, + as2rel_secs: u64, + rpki_secs: u64, + pfx2as_secs: u64, +} + #[derive(Debug, Serialize)] struct ServerDefaults { address: String, @@ -108,13 +133,13 @@ struct FileInfo { pub fn run(config: &MonocleConfig, args: ConfigArgs, output_format: OutputFormat) { match args.command { None => run_status(config, args.verbose, output_format), - Some(ConfigCommands::DbRefresh { + Some(ConfigCommands::Update { asinfo, as2rel, rpki, pfx2as, rtr_endpoint, - }) => run_db_refresh( + }) => run_update( config, asinfo, as2rel, @@ -123,10 +148,10 @@ pub fn run(config: &MonocleConfig, args: ConfigArgs, output_format: OutputFormat rtr_endpoint, output_format, ), - Some(ConfigCommands::DbBackup { destination }) => { - run_db_backup(config, &destination, output_format) + Some(ConfigCommands::Backup { destination }) => { + run_backup(config, &destination, output_format) } - Some(ConfigCommands::DbSources) => run_db_sources(config, output_format), + Some(ConfigCommands::Sources) => run_sources(config, output_format), } } @@ -148,8 +173,15 @@ fn run_status(config: &MonocleConfig, verbose: bool, output_format: OutputFormat let config_info = ConfigInfo { config_file, data_dir: config.data_dir.clone(), + cache_ttl: CacheTtlConfig { + asinfo_secs: config.asinfo_cache_ttl_secs, + as2rel_secs: config.as2rel_cache_ttl_secs, + rpki_secs: config.rpki_cache_ttl_secs, + pfx2as_secs: config.pfx2as_cache_ttl_secs, + }, database: database_info, server_defaults, + rtr_endpoint: config.rtr_endpoint().map(|(h, p)| format!("{}:{}", h, p)), files, }; @@ -211,6 +243,28 @@ fn print_config_table(info: &ConfigInfo, verbose: bool) { println!(" Data dir: {}", info.data_dir); println!(); + println!("Cache TTL:"); + println!( + " ASInfo: {}", + format_duration(info.cache_ttl.asinfo_secs) + ); + println!( + " AS2Rel: {}", + format_duration(info.cache_ttl.as2rel_secs) + ); + println!( + " RPKI: {}", + format_duration(info.cache_ttl.rpki_secs) + ); + println!( + " Pfx2as: {}", + format_duration(info.cache_ttl.pfx2as_secs) + ); + if let Some(ref endpoint) = info.rtr_endpoint { + println!(" RTR endpoint: {}", endpoint); + } + println!(); + println!("Database:"); println!(" Path: {}", info.database.path); println!( @@ -346,15 +400,15 @@ fn print_config_table(info: &ConfigInfo, verbose: bool) { eprintln!(" Use --verbose (-v) to see all files in the data directory"); eprintln!(" Use --format json for machine-readable output"); eprintln!(" Edit ~/.monocle/monocle.toml to customize settings"); - eprintln!(" Use 'monocle config db-sources' to see data source status"); - eprintln!(" Use 'monocle config db-refresh' to refresh data sources"); + eprintln!(" Use 'monocle config sources' to see data source status"); + eprintln!(" Use 'monocle config update' to update data sources"); } // ============================================================================= -// db-refresh subcommand +// update subcommand // ============================================================================= -fn run_db_refresh( +fn run_update( config: &MonocleConfig, asinfo: bool, as2rel: bool, @@ -363,10 +417,10 @@ fn run_db_refresh( rtr_endpoint: Option, output_format: OutputFormat, ) { - // If no specific flags are set, refresh all - let refresh_all = !asinfo && !as2rel && !rpki && !pfx2as; + // If no specific flags are set, update all + let update_all = !asinfo && !as2rel && !rpki && !pfx2as; - let sources_to_refresh: Vec = if refresh_all { + let sources_to_update: Vec = if update_all { DataSource::all() } else { let mut sources = Vec::new(); @@ -385,22 +439,22 @@ fn run_db_refresh( sources }; - refresh_sources( + update_sources( config, - &sources_to_refresh, + &sources_to_update, rtr_endpoint.as_deref(), output_format, ); } #[derive(Debug, Serialize)] -struct RefreshResult { +struct UpdateResult { source: String, result: String, duration_secs: f64, } -fn refresh_sources( +fn update_sources( config: &MonocleConfig, sources: &[DataSource], rtr_endpoint: Option<&str>, @@ -420,10 +474,10 @@ fn refresh_sources( let mut results = Vec::new(); for source in sources { - eprintln!("[monocle] Refreshing {}...", source.name()); + eprintln!("[monocle] Updating {}...", source.name()); let start = Instant::now(); - let result = do_refresh(&db, source, config, rtr_endpoint); + let result = do_update(&db, source, config, rtr_endpoint); let duration = start.elapsed().as_secs_f64(); let result_str = match &result { @@ -437,7 +491,7 @@ fn refresh_sources( } }; - results.push(RefreshResult { + results.push(UpdateResult { source: source.name().to_string(), result: result_str, duration_secs: duration, @@ -463,11 +517,11 @@ fn refresh_sources( } } else { eprintln!(); - eprintln!("[monocle] Refresh completed."); + eprintln!("[monocle] Update completed."); } } -fn do_refresh( +fn do_update( db: &MonocleDatabase, source: &DataSource, config: &MonocleConfig, @@ -477,7 +531,7 @@ fn do_refresh( DataSource::Asinfo => { // Use the database's bootstrap_asinfo method with the passed db connection let counts = db - .bootstrap_asinfo() + .refresh_asinfo() .map_err(|e| format!("Failed to refresh asinfo: {}", e))?; Ok(format!( @@ -488,7 +542,7 @@ fn do_refresh( DataSource::As2rel => { // Use the database's update_as2rel method with the passed db connection let count = db - .update_as2rel() + .refresh_as2rel() .map_err(|e| format!("Failed to refresh as2rel: {}", e))?; Ok(format!("Stored {} relationship entries", count)) @@ -603,10 +657,10 @@ fn do_refresh( } // ============================================================================= -// db-backup subcommand +// backup subcommand // ============================================================================= -fn run_db_backup(config: &MonocleConfig, destination: &str, output_format: OutputFormat) { +fn run_backup(config: &MonocleConfig, destination: &str, output_format: OutputFormat) { let sqlite_path = config.sqlite_path(); let dest_path = Path::new(destination); @@ -672,10 +726,10 @@ fn run_db_backup(config: &MonocleConfig, destination: &str, output_format: Outpu } // ============================================================================= -// db-sources subcommand +// sources subcommand // ============================================================================= -fn run_db_sources(config: &MonocleConfig, output_format: OutputFormat) { +fn run_sources(config: &MonocleConfig, output_format: OutputFormat) { let sources = get_data_source_info(config); if output_format.is_json() { @@ -695,10 +749,10 @@ fn run_db_sources(config: &MonocleConfig, output_format: OutputFormat) { println!("Data Sources:"); println!(); println!( - " {:<15} {:<45} {:<15} Last Updated", - "Name", "Description", "Status" + " {:<12} {:<15} {:<10} Last Updated", + "Name", "Status", "Stale" ); - println!(" {}", "-".repeat(95)); + println!(" {}", "-".repeat(60)); for source in &sources { let status_str = match source.status { @@ -713,19 +767,76 @@ fn run_db_sources(config: &MonocleConfig, output_format: OutputFormat) { DataSourceStatus::NotInitialized => "not initialized".to_string(), }; - let updated_str = source.last_updated.as_deref().unwrap_or("-"); + let updated_str = source + .last_updated + .as_deref() + .map(to_relative_time) + .unwrap_or_else(|| "-".to_string()); + + let stale_str = if source.is_stale { "yes" } else { "no" }; println!( - " {:<15} {:<45} {:<15} {}", - source.name, source.description, status_str, updated_str + " {:<12} {:<15} {:<10} {}", + source.name, status_str, stale_str, updated_str ); } + // Configuration section + println!(); + println!("Configuration:"); + println!( + " ASInfo cache TTL: {}", + format_duration(config.asinfo_cache_ttl_secs) + ); + println!( + " AS2Rel cache TTL: {}", + format_duration(config.as2rel_cache_ttl_secs) + ); + println!( + " RPKI cache TTL: {}", + format_duration(config.rpki_cache_ttl_secs) + ); + println!( + " Pfx2as cache TTL: {}", + format_duration(config.pfx2as_cache_ttl_secs) + ); + if let Some((host, port)) = config.rtr_endpoint() { + println!(" RTR endpoint: {}:{}", host, port); + } + println!(); println!("Usage:"); - println!(" monocle config db-refresh Refresh all data sources"); - println!(" monocle config db-refresh --rpki Refresh only RPKI data"); - println!(" monocle config db-refresh --asinfo Refresh only ASInfo data"); - println!(" monocle config db-backup Backup database to path"); + println!(" monocle config update Update all data sources"); + println!(" monocle config update --rpki Update only RPKI data"); + println!(" monocle config update --asinfo Update only ASInfo data"); + println!(" monocle config backup Backup database to path"); + } +} + +/// Format duration in seconds to human-readable string +fn format_duration(secs: u64) -> String { + if secs >= 86400 { + let days = secs / 86400; + if days == 1 { + "1 day".to_string() + } else { + format!("{} days", days) + } + } else if secs >= 3600 { + let hours = secs / 3600; + if hours == 1 { + "1 hour".to_string() + } else { + format!("{} hours", hours) + } + } else if secs >= 60 { + let mins = secs / 60; + if mins == 1 { + "1 minute".to_string() + } else { + format!("{} minutes", mins) + } + } else { + format!("{} seconds", secs) } } diff --git a/src/bin/commands/inspect.rs b/src/bin/commands/inspect.rs index 7390c64..8a2f011 100644 --- a/src/bin/commands/inspect.rs +++ b/src/bin/commands/inspect.rs @@ -81,7 +81,7 @@ pub fn run( config: &MonocleConfig, args: InspectArgs, output_format: OutputFormat, - no_refresh: bool, + no_update: bool, ) { let sqlite_path = config.sqlite_path(); @@ -94,12 +94,12 @@ pub fn run( } }; - let lens = InspectLens::new(&db); + let lens = InspectLens::new(&db, config); // Handle explicit update request (force refresh all) if args.update { - if no_refresh { - eprintln!("[monocle] Warning: --update ignored because --no-refresh is set"); + if no_update { + eprintln!("[monocle] Warning: --update ignored because --no-update is set"); } else { eprintln!("[monocle] Updating all data sources..."); match lens.ensure_data_available() { @@ -131,8 +131,8 @@ pub fn run( let required_sections = determine_required_sections(&args, &options, &lens); // Ensure only the required data sources are available (auto-refresh if empty or expired) - // Skip if --no-refresh is set - if !no_refresh { + // Skip if --no-update is set + if !no_update { match lens.ensure_data_for_sections(&required_sections) { Ok(summary) => { // Print messages about any data that was refreshed diff --git a/src/bin/commands/pfx2as.rs b/src/bin/commands/pfx2as.rs index cd39dd4..3f715c2 100644 --- a/src/bin/commands/pfx2as.rs +++ b/src/bin/commands/pfx2as.rs @@ -58,12 +58,7 @@ impl From<&Pfx2asArgs> for Pfx2asSearchArgs { } } -pub fn run( - config: &MonocleConfig, - args: Pfx2asArgs, - output_format: OutputFormat, - no_refresh: bool, -) { +pub fn run(config: &MonocleConfig, args: Pfx2asArgs, output_format: OutputFormat, no_update: bool) { let sqlite_path = config.sqlite_path(); // Open the database @@ -79,8 +74,8 @@ pub fn run( // Handle explicit updates if args.update { - if no_refresh { - eprintln!("[monocle] Warning: --update ignored because --no-refresh is set"); + if no_update { + eprintln!("[monocle] Warning: --update ignored because --no-update is set"); } else { eprintln!("[monocle] Updating pfx2as data..."); @@ -97,8 +92,8 @@ pub fn run( } // Check if pfx2as data needs refresh - if !no_refresh { - match lens.refresh_reason() { + if !no_update { + match lens.refresh_reason(config.pfx2as_cache_ttl()) { Ok(Some(reason)) => { eprintln!("[monocle] Pfx2as {}, updating now...", reason); match lens.refresh(None) { @@ -122,7 +117,7 @@ pub fn run( // Also ensure RPKI data is available for validation let rpki_lens = RpkiLens::new(&db); - if let Ok(Some(reason)) = rpki_lens.refresh_reason() { + if let Ok(Some(reason)) = rpki_lens.refresh_reason(config.rpki_cache_ttl()) { eprintln!("[monocle] RPKI {}, updating for validation...", reason); match rpki_lens.refresh() { Ok((roa_count, aspa_count)) => { diff --git a/src/bin/commands/rpki.rs b/src/bin/commands/rpki.rs index f57e07c..64e4fb8 100644 --- a/src/bin/commands/rpki.rs +++ b/src/bin/commands/rpki.rs @@ -6,6 +6,7 @@ use monocle::lens::rpki::{ RpkiRoaLookupArgs, RpkiViewsCollectorOption, }; use monocle::lens::utils::OutputFormat; +use monocle::MonocleConfig; use std::collections::HashSet; use tabled::settings::object::Columns; use tabled::settings::width::Width; @@ -76,16 +77,21 @@ pub enum RpkiCommands { }, } -pub fn run(commands: RpkiCommands, output_format: OutputFormat, data_dir: &str, no_refresh: bool) { +pub fn run( + commands: RpkiCommands, + output_format: OutputFormat, + config: &MonocleConfig, + no_update: bool, +) { match commands { RpkiCommands::Validate { resources, refresh } => { - let effective_refresh = if no_refresh && refresh { - eprintln!("[monocle] Warning: --refresh ignored because --no-refresh is set"); + let effective_refresh = if no_update && refresh { + eprintln!("[monocle] Warning: --refresh ignored because --no-update is set"); false } else { refresh }; - run_validate(resources, effective_refresh, output_format, data_dir) + run_validate(resources, effective_refresh, output_format, config) } RpkiCommands::Roas { resources, @@ -94,8 +100,8 @@ pub fn run(commands: RpkiCommands, output_format: OutputFormat, data_dir: &str, collector, refresh, } => { - let effective_refresh = if no_refresh && refresh { - eprintln!("[monocle] Warning: --refresh ignored because --no-refresh is set"); + let effective_refresh = if no_update && refresh { + eprintln!("[monocle] Warning: --refresh ignored because --no-update is set"); false } else { refresh @@ -107,8 +113,8 @@ pub fn run(commands: RpkiCommands, output_format: OutputFormat, data_dir: &str, collector, effective_refresh, output_format, - data_dir, - no_refresh, + config, + no_update, ) } RpkiCommands::Aspas { @@ -119,8 +125,8 @@ pub fn run(commands: RpkiCommands, output_format: OutputFormat, data_dir: &str, collector, refresh, } => { - let effective_refresh = if no_refresh && refresh { - eprintln!("[monocle] Warning: --refresh ignored because --no-refresh is set"); + let effective_refresh = if no_update && refresh { + eprintln!("[monocle] Warning: --refresh ignored because --no-update is set"); false } else { refresh @@ -133,8 +139,8 @@ pub fn run(commands: RpkiCommands, output_format: OutputFormat, data_dir: &str, collector, effective_refresh, output_format, - data_dir, - no_refresh, + config, + no_update, ) } } @@ -177,10 +183,11 @@ enum ResourceType { fn ensure_rpki_cache( lens: &RpkiLens, force_refresh: bool, + ttl: std::time::Duration, ) -> Result<(), Box> { // Check reason for refresh let refresh_reason = lens - .refresh_reason() + .refresh_reason(ttl) .map_err(|e| format!("Failed to check cache: {}", e))?; let needs_refresh = force_refresh || refresh_reason.is_some(); @@ -208,17 +215,17 @@ fn ensure_rpki_cache( /// Ensure ASInfo data is available for enriching ASPA output fn ensure_asinfo_for_aspa( db: &MonocleDatabase, - no_refresh: bool, + no_update: bool, ) -> Result<(), Box> { if db.asinfo().is_empty() { - if no_refresh { + if no_update { eprintln!("[monocle] Warning: ASInfo data is empty. AS names will not be shown."); - eprintln!("[monocle] Run without --no-refresh or use 'monocle config db-refresh --asinfo' to load data."); + eprintln!("[monocle] Run without --no-update or use 'monocle config update --asinfo' to load data."); return Ok(()); } eprintln!("[monocle] Loading ASInfo data for AS name enrichment..."); let counts = db - .bootstrap_asinfo() + .refresh_asinfo() .map_err(|e| format!("Failed to load ASInfo data: {}", e))?; eprintln!( "[monocle] Loaded {} core, {} as2org records", @@ -232,7 +239,7 @@ fn run_validate( resources: Vec, refresh: bool, output_format: OutputFormat, - data_dir: &str, + config: &MonocleConfig, ) { if resources.len() != 2 { eprintln!( @@ -286,7 +293,7 @@ fn run_validate( }; // Open database and create lens - let db = match MonocleDatabase::open_in_dir(data_dir) { + let db = match MonocleDatabase::open_in_dir(&config.data_dir) { Ok(db) => db, Err(e) => { eprintln!("ERROR: Failed to open database: {}", e); @@ -295,7 +302,7 @@ fn run_validate( }; let lens = RpkiLens::new(&db); - if let Err(e) = ensure_rpki_cache(&lens, refresh) { + if let Err(e) = ensure_rpki_cache(&lens, refresh, config.rpki_cache_ttl()) { eprintln!("ERROR: Failed to refresh RPKI cache: {}", e); return; } @@ -434,8 +441,8 @@ fn run_roas( collector: String, refresh: bool, output_format: OutputFormat, - data_dir: &str, - no_refresh: bool, + config: &MonocleConfig, + no_update: bool, ) { // Parse date if provided let (parsed_date, date_str) = match &date { @@ -448,7 +455,7 @@ fn run_roas( }, None => { // For current data (no date), use SQLite cache - run_roas_from_cache(resources, refresh, output_format, data_dir, no_refresh); + run_roas_from_cache(resources, refresh, output_format, config, no_update); return; } }; @@ -478,7 +485,7 @@ fn run_roas( } // For historical data, we need a database reference but won't use its cache - let db = match MonocleDatabase::open_in_dir(data_dir) { + let db = match MonocleDatabase::open_in_dir(&config.data_dir) { Ok(db) => db, Err(e) => { eprintln!("ERROR: Failed to open database: {}", e); @@ -588,11 +595,11 @@ fn run_roas_from_cache( resources: Vec, refresh: bool, output_format: OutputFormat, - data_dir: &str, - no_refresh: bool, + config: &MonocleConfig, + no_update: bool, ) { // Open database and create lens - let db = match MonocleDatabase::open_in_dir(data_dir) { + let db = match MonocleDatabase::open_in_dir(&config.data_dir) { Ok(db) => db, Err(e) => { eprintln!("ERROR: Failed to open database: {}", e); @@ -601,8 +608,8 @@ fn run_roas_from_cache( }; let lens = RpkiLens::new(&db); - if !no_refresh { - if let Err(e) = ensure_rpki_cache(&lens, refresh) { + if !no_update { + if let Err(e) = ensure_rpki_cache(&lens, refresh, config.rpki_cache_ttl()) { eprintln!("ERROR: Failed to refresh RPKI cache: {}", e); return; } @@ -841,8 +848,8 @@ fn run_aspas( collector: String, refresh: bool, output_format: OutputFormat, - data_dir: &str, - no_refresh: bool, + config: &MonocleConfig, + no_update: bool, ) { // Parse date if provided let (parsed_date, date_str) = match &date { @@ -860,8 +867,8 @@ fn run_aspas( provider, refresh, output_format, - data_dir, - no_refresh, + config, + no_update, ); return; } @@ -877,7 +884,7 @@ fn run_aspas( eprintln!("{}", source_display); // For historical data, we need a database reference but won't use its cache - let db = match MonocleDatabase::open_in_dir(data_dir) { + let db = match MonocleDatabase::open_in_dir(&config.data_dir) { Ok(db) => db, Err(e) => { eprintln!("ERROR: Failed to open database: {}", e); @@ -919,11 +926,11 @@ fn run_aspas_from_cache( provider: Option, refresh: bool, output_format: OutputFormat, - data_dir: &str, - no_refresh: bool, + config: &MonocleConfig, + no_update: bool, ) { // Open database and create lens - let db = match MonocleDatabase::open_in_dir(data_dir) { + let db = match MonocleDatabase::open_in_dir(&config.data_dir) { Ok(db) => db, Err(e) => { eprintln!("ERROR: Failed to open database: {}", e); @@ -932,14 +939,14 @@ fn run_aspas_from_cache( }; // Ensure ASInfo data is available for AS name enrichment - if let Err(e) = ensure_asinfo_for_aspa(&db, no_refresh) { + if let Err(e) = ensure_asinfo_for_aspa(&db, no_update) { eprintln!("ERROR: {}", e); return; } let lens = RpkiLens::new(&db); - if !no_refresh { - if let Err(e) = ensure_rpki_cache(&lens, refresh) { + if !no_update { + if let Err(e) = ensure_rpki_cache(&lens, refresh, config.rpki_cache_ttl()) { eprintln!("ERROR: Failed to refresh RPKI cache: {}", e); return; } diff --git a/src/bin/monocle.rs b/src/bin/monocle.rs index e3f224f..830cf94 100644 --- a/src/bin/monocle.rs +++ b/src/bin/monocle.rs @@ -41,9 +41,9 @@ struct Cli { #[clap(long, global = true)] json: bool, - /// Disable automatic data refresh (use existing cached data only) + /// Disable automatic database updates (use existing cached data only) #[clap(long, global = true)] - no_refresh: bool, + no_update: bool, #[clap(subcommand)] command: Commands, @@ -182,10 +182,14 @@ fn main() { // binary as the entrypoint, but compile this arm only when `server` is enabled. #[cfg(feature = "cli")] { - let data_dir = args.data_dir.unwrap_or_else(|| config.data_dir.clone()); + // Create context from config, optionally overriding the data directory + let mut server_config = config.clone(); + if let Some(data_dir) = args.data_dir { + server_config.data_dir = data_dir; + } let router = monocle::server::create_router(); - let context = monocle::server::WsContext::new(data_dir); + let context = monocle::server::WsContext::from_config(server_config); let mut server_config = monocle::server::ServerConfig::default() .with_address(args.address) @@ -231,19 +235,19 @@ fn main() { } Commands::Inspect(args) => { - commands::inspect::run(&config, args, output_format, cli.no_refresh) + commands::inspect::run(&config, args, output_format, cli.no_update) } Commands::Time(args) => commands::time::run(args, output_format), Commands::Country(args) => commands::country::run(args, output_format), Commands::Rpki { commands } => { - commands::rpki::run(commands, output_format, &config.data_dir, cli.no_refresh) + commands::rpki::run(commands, output_format, &config, cli.no_update) } Commands::Ip(args) => commands::ip::run(args, output_format), Commands::As2rel(args) => { - commands::as2rel::run(&config, args, output_format, cli.no_refresh) + commands::as2rel::run(&config, args, output_format, cli.no_update) } Commands::Pfx2as(args) => { - commands::pfx2as::run(&config, args, output_format, cli.no_refresh) + commands::pfx2as::run(&config, args, output_format, cli.no_update) } Commands::Config(args) => commands::config::run(&config, args, output_format), } diff --git a/src/config.rs b/src/config.rs index b330108..09f34f7 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,14 +4,24 @@ use serde::Serialize; use std::collections::HashMap; use std::path::Path; +/// Default TTL for all data sources: 7 days in seconds +pub const DEFAULT_CACHE_TTL_SECS: u64 = 604800; + +#[derive(Clone)] pub struct MonocleConfig { /// Path to the directory to hold Monocle's data pub data_dir: String, - /// TTL for RPKI cache in seconds (default: 1 hour) + /// TTL for ASInfo cache in seconds (default: 7 days) + pub asinfo_cache_ttl_secs: u64, + + /// TTL for AS2Rel cache in seconds (default: 7 days) + pub as2rel_cache_ttl_secs: u64, + + /// TTL for RPKI cache in seconds (default: 7 days) pub rpki_cache_ttl_secs: u64, - /// TTL for Pfx2as cache in seconds (default: 24 hours) + /// TTL for Pfx2as cache in seconds (default: 7 days) pub pfx2as_cache_ttl_secs: u64, /// RTR server hostname (optional) @@ -34,8 +44,11 @@ const EMPTY_CONFIG: &str = r#"### monocle configuration file # data_dir = "~/.monocle" ### cache TTL settings (in seconds) -# rpki_cache_ttl_secs = 3600 # 1 hour -# pfx2as_cache_ttl_secs = 86400 # 24 hours +### all data sources default to 7 days (604800 seconds) +# asinfo_cache_ttl_secs = 604800 +# as2rel_cache_ttl_secs = 604800 +# rpki_cache_ttl_secs = 604800 +# pfx2as_cache_ttl_secs = 604800 ### RTR endpoint for ROA data (optional) ### If set, ROAs will be fetched via RTR protocol instead of Cloudflare JSON API @@ -55,8 +68,10 @@ impl Default for MonocleConfig { Self { data_dir: format!("{}/.monocle", home_dir), - rpki_cache_ttl_secs: 3600, // 1 hour - pfx2as_cache_ttl_secs: 86400, // 24 hours + asinfo_cache_ttl_secs: DEFAULT_CACHE_TTL_SECS, + as2rel_cache_ttl_secs: DEFAULT_CACHE_TTL_SECS, + rpki_cache_ttl_secs: DEFAULT_CACHE_TTL_SECS, + pfx2as_cache_ttl_secs: DEFAULT_CACHE_TTL_SECS, rpki_rtr_host: None, rpki_rtr_port: 8282, rpki_rtr_timeout_secs: 10, @@ -141,17 +156,29 @@ impl MonocleConfig { } }; - // Parse RPKI cache TTL (default: 1 hour) + // Parse ASInfo cache TTL (default: 7 days) + let asinfo_cache_ttl_secs = config + .get("asinfo_cache_ttl_secs") + .and_then(|s| s.parse().ok()) + .unwrap_or(DEFAULT_CACHE_TTL_SECS); + + // Parse AS2Rel cache TTL (default: 7 days) + let as2rel_cache_ttl_secs = config + .get("as2rel_cache_ttl_secs") + .and_then(|s| s.parse().ok()) + .unwrap_or(DEFAULT_CACHE_TTL_SECS); + + // Parse RPKI cache TTL (default: 7 days) let rpki_cache_ttl_secs = config .get("rpki_cache_ttl_secs") .and_then(|s| s.parse().ok()) - .unwrap_or(3600); + .unwrap_or(DEFAULT_CACHE_TTL_SECS); - // Parse Pfx2as cache TTL (default: 24 hours) + // Parse Pfx2as cache TTL (default: 7 days) let pfx2as_cache_ttl_secs = config .get("pfx2as_cache_ttl_secs") .and_then(|s| s.parse().ok()) - .unwrap_or(86400); + .unwrap_or(DEFAULT_CACHE_TTL_SECS); // Parse RTR configuration let rpki_rtr_host = config.get("rpki_rtr_host").cloned(); @@ -170,6 +197,8 @@ impl MonocleConfig { Ok(MonocleConfig { data_dir, + asinfo_cache_ttl_secs, + as2rel_cache_ttl_secs, rpki_cache_ttl_secs, pfx2as_cache_ttl_secs, rpki_rtr_host, @@ -185,6 +214,16 @@ impl MonocleConfig { format!("{}/monocle-data.sqlite3", data_dir) } + /// Get ASInfo cache TTL as Duration + pub fn asinfo_cache_ttl(&self) -> std::time::Duration { + std::time::Duration::from_secs(self.asinfo_cache_ttl_secs) + } + + /// Get AS2Rel cache TTL as Duration + pub fn as2rel_cache_ttl(&self) -> std::time::Duration { + std::time::Duration::from_secs(self.as2rel_cache_ttl_secs) + } + /// Get RPKI cache TTL as Duration pub fn rpki_cache_ttl(&self) -> std::time::Duration { std::time::Duration::from_secs(self.rpki_cache_ttl_secs) @@ -217,6 +256,8 @@ impl MonocleConfig { let mut lines = vec![ format!("Data Directory: {}", self.data_dir), format!("SQLite Path: {}", self.sqlite_path()), + format!("ASInfo Cache TTL: {} seconds", self.asinfo_cache_ttl_secs), + format!("AS2Rel Cache TTL: {} seconds", self.as2rel_cache_ttl_secs), format!("RPKI Cache TTL: {} seconds", self.rpki_cache_ttl_secs), format!("Pfx2as Cache TTL: {} seconds", self.pfx2as_cache_ttl_secs), ]; @@ -263,6 +304,11 @@ pub struct DataSourceInfo { #[serde(skip_serializing_if = "Option::is_none")] pub last_updated: Option, pub status: DataSourceStatus, + /// Whether this data source currently needs to be refreshed (stale or empty) + pub is_stale: bool, + /// Configured TTL in seconds (None for sources that don't expire) + #[serde(skip_serializing_if = "Option::is_none")] + pub ttl_secs: Option, } /// Status of a data source @@ -320,6 +366,8 @@ pub struct SqliteDatabaseInfo { /// Cache settings #[derive(Debug, Serialize, Clone)] pub struct CacheSettings { + pub asinfo_ttl_secs: u64, + pub as2rel_ttl_secs: u64, pub rpki_ttl_secs: u64, pub pfx2as_ttl_secs: u64, } @@ -385,7 +433,7 @@ impl std::fmt::Display for DataSource { } /// Get SQLite database information -#[cfg(feature = "database")] +#[cfg(feature = "lib")] pub fn get_sqlite_info(config: &MonocleConfig) -> SqliteDatabaseInfo { use crate::database::{MonocleDatabase, SchemaManager, SchemaStatus, SCHEMA_VERSION}; @@ -530,15 +578,28 @@ pub fn get_sqlite_info(config: &MonocleConfig) -> SqliteDatabaseInfo { /// Get cache settings pub fn get_cache_settings(config: &MonocleConfig) -> CacheSettings { CacheSettings { + asinfo_ttl_secs: config.asinfo_cache_ttl_secs, + as2rel_ttl_secs: config.as2rel_cache_ttl_secs, rpki_ttl_secs: config.rpki_cache_ttl_secs, pfx2as_ttl_secs: config.pfx2as_cache_ttl_secs, } } /// Get detailed information about all data sources -#[cfg(feature = "database")] +#[cfg(feature = "lib")] pub fn get_data_source_info(config: &MonocleConfig) -> Vec { + use crate::database::MonocleDatabase; + use std::time::Duration; + let sqlite_info = get_sqlite_info(config); + let sqlite_path = config.sqlite_path(); + + // Try to open the database to check staleness + let db = if sqlite_info.exists { + MonocleDatabase::open(&sqlite_path).ok() + } else { + None + }; let mut sources = Vec::new(); @@ -548,12 +609,19 @@ pub fn get_data_source_info(config: &MonocleConfig) -> Vec { Some(_) => DataSourceStatus::Empty, None => DataSourceStatus::NotInitialized, }; + let asinfo_ttl = Duration::from_secs(config.asinfo_cache_ttl_secs); + let asinfo_is_stale = db + .as_ref() + .map(|d| d.asinfo().needs_refresh(asinfo_ttl)) + .unwrap_or(true); sources.push(DataSourceInfo { name: DataSource::Asinfo.name().to_string(), description: DataSource::Asinfo.description().to_string(), record_count: sqlite_info.asinfo_count, last_updated: sqlite_info.asinfo_last_updated.clone(), status: asinfo_status, + is_stale: asinfo_is_stale, + ttl_secs: Some(config.asinfo_cache_ttl_secs), }); // AS2Rel @@ -562,15 +630,22 @@ pub fn get_data_source_info(config: &MonocleConfig) -> Vec { Some(_) => DataSourceStatus::Empty, None => DataSourceStatus::NotInitialized, }; + let as2rel_ttl = Duration::from_secs(config.as2rel_cache_ttl_secs); + let as2rel_is_stale = db + .as_ref() + .map(|d| d.as2rel().needs_refresh(as2rel_ttl)) + .unwrap_or(true); sources.push(DataSourceInfo { name: DataSource::As2rel.name().to_string(), description: DataSource::As2rel.description().to_string(), record_count: sqlite_info.as2rel_count, last_updated: sqlite_info.as2rel_last_updated.clone(), status: as2rel_status, + is_stale: as2rel_is_stale, + ttl_secs: Some(config.as2rel_cache_ttl_secs), }); - // RPKI (combined ROA + ASPA count for record_count, but we'll show details separately) + // RPKI let rpki_total = match (sqlite_info.rpki_roa_count, sqlite_info.rpki_aspa_count) { (Some(roa), Some(aspa)) => Some(roa + aspa), (Some(roa), None) => Some(roa), @@ -582,12 +657,19 @@ pub fn get_data_source_info(config: &MonocleConfig) -> Vec { Some(_) => DataSourceStatus::Empty, None => DataSourceStatus::NotInitialized, }; + let rpki_ttl = Duration::from_secs(config.rpki_cache_ttl_secs); + let rpki_is_stale = db + .as_ref() + .map(|d| d.rpki().needs_refresh(rpki_ttl)) + .unwrap_or(true); sources.push(DataSourceInfo { name: DataSource::Rpki.name().to_string(), description: DataSource::Rpki.description().to_string(), record_count: rpki_total, last_updated: sqlite_info.rpki_last_updated.clone(), status: rpki_status, + is_stale: rpki_is_stale, + ttl_secs: Some(config.rpki_cache_ttl_secs), }); // Pfx2as @@ -596,12 +678,19 @@ pub fn get_data_source_info(config: &MonocleConfig) -> Vec { Some(_) => DataSourceStatus::Empty, None => DataSourceStatus::NotInitialized, }; + let pfx2as_ttl = Duration::from_secs(config.pfx2as_cache_ttl_secs); + let pfx2as_is_stale = db + .as_ref() + .map(|d| d.pfx2as().needs_refresh(pfx2as_ttl)) + .unwrap_or(true); sources.push(DataSourceInfo { name: DataSource::Pfx2as.name().to_string(), description: DataSource::Pfx2as.description().to_string(), record_count: sqlite_info.pfx2as_count, last_updated: sqlite_info.pfx2as_last_updated.clone(), status: pfx2as_status, + is_stale: pfx2as_is_stale, + ttl_secs: Some(config.pfx2as_cache_ttl_secs), }); sources @@ -631,8 +720,10 @@ mod tests { #[test] fn test_default_config() { let config = MonocleConfig::default(); - assert_eq!(config.rpki_cache_ttl_secs, 3600); - assert_eq!(config.pfx2as_cache_ttl_secs, 86400); + assert_eq!(config.asinfo_cache_ttl_secs, DEFAULT_CACHE_TTL_SECS); // 7 days + assert_eq!(config.as2rel_cache_ttl_secs, DEFAULT_CACHE_TTL_SECS); // 7 days + assert_eq!(config.rpki_cache_ttl_secs, DEFAULT_CACHE_TTL_SECS); // 7 days + assert_eq!(config.pfx2as_cache_ttl_secs, DEFAULT_CACHE_TTL_SECS); // 7 days assert_eq!(config.rpki_rtr_host, None); assert_eq!(config.rpki_rtr_port, 8282); assert_eq!(config.rpki_rtr_timeout_secs, 10); @@ -643,6 +734,8 @@ mod tests { fn test_paths() { let config = MonocleConfig { data_dir: "/test/dir".to_string(), + asinfo_cache_ttl_secs: DEFAULT_CACHE_TTL_SECS, + as2rel_cache_ttl_secs: DEFAULT_CACHE_TTL_SECS, rpki_cache_ttl_secs: 3600, pfx2as_cache_ttl_secs: 86400, rpki_rtr_host: None, @@ -659,6 +752,8 @@ mod tests { fn test_ttl_durations() { let config = MonocleConfig { data_dir: "/test".to_string(), + asinfo_cache_ttl_secs: 1000, + as2rel_cache_ttl_secs: 2000, rpki_cache_ttl_secs: 7200, pfx2as_cache_ttl_secs: 3600, rpki_rtr_host: None, @@ -667,6 +762,14 @@ mod tests { rpki_rtr_no_fallback: false, }; + assert_eq!( + config.asinfo_cache_ttl(), + std::time::Duration::from_secs(1000) + ); + assert_eq!( + config.as2rel_cache_ttl(), + std::time::Duration::from_secs(2000) + ); assert_eq!( config.rpki_cache_ttl(), std::time::Duration::from_secs(7200) @@ -687,6 +790,8 @@ mod tests { // RTR configured let config = MonocleConfig { data_dir: "/test".to_string(), + asinfo_cache_ttl_secs: DEFAULT_CACHE_TTL_SECS, + as2rel_cache_ttl_secs: DEFAULT_CACHE_TTL_SECS, rpki_cache_ttl_secs: 3600, pfx2as_cache_ttl_secs: 86400, rpki_rtr_host: Some("rtr.example.com".to_string()), diff --git a/src/database/README.md b/src/database/README.md index eb433d1..75cfa0c 100644 --- a/src/database/README.md +++ b/src/database/README.md @@ -88,8 +88,8 @@ use monocle::database::MonocleDatabase; let db = MonocleDatabase::open_in_dir("~/.monocle")?; // Bootstrap ASInfo data if needed -if db.needs_asinfo_bootstrap() { - let count = db.bootstrap_asinfo()?; +if db.needs_asinfo_refresh(Duration::from_secs(7 * 24 * 60 * 60)) { + let count = db.refresh_asinfo()?; println!("Loaded {} ASes", count); } @@ -100,7 +100,9 @@ for r in results { } // Update AS2Rel data -if db.needs_as2rel_update() { +use std::time::Duration; +let ttl = Duration::from_secs(24 * 60 * 60); // 24 hours +if db.needs_as2rel_refresh(ttl) { let count = db.update_as2rel()?; println!("Loaded {} relationships", count); } @@ -144,7 +146,7 @@ let pfx2as = db.pfx2as(); // Check if refresh is needed if pfx2as.needs_refresh(DEFAULT_PFX2AS_CACHE_TTL)? { - // Refresh via CLI: monocle config db-refresh pfx2as + // Refresh via CLI: monocle config update --pfx2as // Or via WebSocket: database.refresh with source: "pfx2as" } diff --git a/src/database/mod.rs b/src/database/mod.rs index e9b9379..84cadf1 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -53,8 +53,8 @@ //! let db = MonocleDatabase::open_in_dir("~/.monocle")?; //! //! // Bootstrap ASInfo data if needed -//! if db.needs_asinfo_bootstrap() { -//! db.bootstrap_asinfo()?; +//! if db.needs_asinfo_refresh(Duration::from_secs(7 * 24 * 60 * 60)) { +//! db.refresh_asinfo()?; //! } //! //! // Query AS data @@ -122,8 +122,8 @@ pub use monocle::{ }; // Session types (SQLite-based for search result exports) -// Requires lens-bgpkit feature because MsgStore depends on bgpkit_parser::BgpElem -#[cfg(feature = "lens-bgpkit")] +// Requires lib feature because MsgStore depends on bgpkit_parser::BgpElem +#[cfg(feature = "lib")] pub use session::MsgStore; // ============================================================================= diff --git a/src/database/monocle/as2rel.rs b/src/database/monocle/as2rel.rs index 99476bd..d6dd183 100644 --- a/src/database/monocle/as2rel.rs +++ b/src/database/monocle/as2rel.rs @@ -12,9 +12,6 @@ use tracing::info; /// Default URL for AS2Rel data pub const BGPKIT_AS2REL_URL: &str = "https://data.bgpkit.com/as2rel/as2rel-latest.json.bz2"; -/// Seven days in seconds (for staleness check) -const SEVEN_DAYS_SECS: u64 = 7 * 24 * 60 * 60; - /// Repository for AS2Rel data operations /// /// Provides methods for querying and updating AS-level relationship data @@ -125,20 +122,20 @@ impl<'a> As2relRepository<'a> { Ok(count) } - /// Check if the data should be updated (empty or older than 7 days) - pub fn should_update(&self) -> bool { + /// Check if data needs refresh based on configurable TTL + pub fn needs_refresh(&self, ttl: std::time::Duration) -> bool { if self.is_empty() { return true; } - // Check if data is older than 7 days match self.get_meta() { Ok(Some(meta)) => { let now = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0); - now.saturating_sub(meta.last_updated) > SEVEN_DAYS_SECS + let age = now.saturating_sub(meta.last_updated); + age >= ttl.as_secs() } _ => true, } @@ -1048,12 +1045,12 @@ mod tests { } #[test] - fn test_should_update() { + fn test_needs_refresh() { let db = setup_test_db(); let repo = As2relRepository::new(&db.conn); - // Empty database should need update - assert!(repo.should_update()); + // Empty database should need refresh + assert!(repo.needs_refresh(std::time::Duration::from_secs(7 * 24 * 60 * 60))); // Insert data with old timestamp db.conn @@ -1070,7 +1067,7 @@ mod tests { ) .unwrap(); - // Old data should need update - assert!(repo.should_update()); + // Old data should need refresh + assert!(repo.needs_refresh(std::time::Duration::from_secs(7 * 24 * 60 * 60))); } } diff --git a/src/database/monocle/asinfo.rs b/src/database/monocle/asinfo.rs index 7327c48..277e2f5 100644 --- a/src/database/monocle/asinfo.rs +++ b/src/database/monocle/asinfo.rs @@ -436,6 +436,35 @@ impl<'a> AsinfoRepository<'a> { self.store_from_jsonl(&records, url) } + /// Load from local file path and store (convenience wrapper) + pub fn load_from_path(&self, path: &str) -> Result { + info!("Loading ASInfo data from {}", path); + + let file = + std::fs::File::open(path).map_err(|e| anyhow!("Failed to open ASInfo file: {}", e))?; + + let buf_reader = std::io::BufReader::new(file); + let mut records = Vec::new(); + + for (line_num, line) in buf_reader.lines().enumerate() { + let line = line.map_err(|e| anyhow!("Failed to read line {}: {}", line_num + 1, e))?; + if line.trim().is_empty() { + continue; + } + + match serde_json::from_str::(&line) { + Ok(record) => records.push(record), + Err(e) => { + // Log warning but continue processing + tracing::warn!("Failed to parse line {}: {}", line_num + 1, e); + } + } + } + + info!("Parsed {} records from JSONL", records.len()); + self.store_from_jsonl(&records, path) + } + /// Clear all asinfo tables pub fn clear(&self) -> Result<()> { self.conn @@ -482,7 +511,7 @@ impl<'a> AsinfoRepository<'a> { Ok(Some(meta)) => { let now = chrono::Utc::now().timestamp(); let age = now - meta.last_updated; - age > ttl.as_secs() as i64 + age >= ttl.as_secs() as i64 } _ => true, } diff --git a/src/database/monocle/mod.rs b/src/database/monocle/mod.rs index fb6777e..4f11c33 100644 --- a/src/database/monocle/mod.rs +++ b/src/database/monocle/mod.rs @@ -34,8 +34,42 @@ pub use rpki::{ use crate::database::core::{DatabaseConn, SchemaManager, SchemaStatus}; use anyhow::Result; +use chrono::{DateTime, Utc}; use tracing::info; +/// Result of a data refresh operation +/// +/// Provides consistent information about refresh operations across all repositories. +#[derive(Debug, Clone)] +pub struct RefreshResult { + /// Number of records loaded + pub records_loaded: usize, + /// Source URL or path + pub source: String, + /// Timestamp when the refresh occurred + pub timestamp: DateTime, + /// Additional details (repository-specific) + pub details: Option, +} + +impl RefreshResult { + /// Create a new refresh result + pub fn new(records_loaded: usize, source: impl Into) -> Self { + Self { + records_loaded, + source: source.into(), + timestamp: Utc::now(), + details: None, + } + } + + /// Add details to the result + pub fn with_details(mut self, details: impl Into) -> Self { + self.details = Some(details.into()); + self + } +} + /// Main monocle database for persistent data (SQLite backend) /// /// `MonocleDatabase` provides a unified interface to all monocle data tables. @@ -136,62 +170,129 @@ impl MonocleDatabase { &self.db.conn } - /// Check if the ASInfo data needs to be bootstrapped - pub fn needs_asinfo_bootstrap(&self) -> bool { + // ========================================================================= + // ASInfo Data Management + // ========================================================================= + + /// Check if ASInfo data is empty (needs initial load) + pub fn needs_asinfo_refresh(&self, _ttl: std::time::Duration) -> bool { self.asinfo().is_empty() } - /// Check if the ASInfo data needs refresh - pub fn needs_asinfo_refresh(&self) -> bool { - self.asinfo().needs_refresh(DEFAULT_ASINFO_TTL) + /// Refresh ASInfo data from the default URL + /// + /// Returns the counts of records loaded per table. + pub fn refresh_asinfo(&self) -> Result { + self.asinfo().load_from_url(ASINFO_DATA_URL) } - /// Bootstrap ASInfo data from the default URL + /// Refresh ASInfo data from a custom path /// /// Returns the counts of records loaded per table. - pub fn bootstrap_asinfo(&self) -> Result { - self.asinfo().load_from_url(ASINFO_DATA_URL) + pub fn refresh_asinfo_from(&self, path: &str) -> Result { + self.asinfo().load_from_path(path) } - /// Check if the AS2Rel data needs to be updated - pub fn needs_as2rel_update(&self) -> bool { - self.as2rel().should_update() + // ========================================================================= + // AS2Rel Data Management + // ========================================================================= + + /// Check if AS2Rel data needs refresh + pub fn needs_as2rel_refresh(&self, ttl: std::time::Duration) -> bool { + self.as2rel().needs_refresh(ttl) } - /// Update AS2Rel data from the default URL + /// Refresh AS2Rel data from the default URL /// /// Returns the number of entries loaded. - pub fn update_as2rel(&self) -> Result { + pub fn refresh_as2rel(&self) -> Result { self.as2rel().load_from_url() } - /// Update AS2Rel data from a custom path + /// Refresh AS2Rel data from a custom path /// /// Returns the number of entries loaded. - pub fn update_as2rel_from(&self, path: &str) -> Result { + pub fn refresh_as2rel_from(&self, path: &str) -> Result { self.as2rel().load_from_path(path) } - /// Check if the RPKI cache needs refresh - pub fn needs_rpki_refresh(&self) -> bool { - self.rpki().needs_refresh(DEFAULT_RPKI_CACHE_TTL) - } + // ========================================================================= + // RPKI Data Management + // ========================================================================= - /// Check if the RPKI cache needs refresh with custom TTL - pub fn needs_rpki_refresh_with_ttl(&self, ttl: chrono::Duration) -> bool { + /// Check if RPKI data needs refresh + pub fn needs_rpki_refresh(&self, ttl: std::time::Duration) -> bool { self.rpki().needs_refresh(ttl) } - /// Check if the Pfx2as cache needs refresh - pub fn needs_pfx2as_refresh(&self) -> bool { - self.pfx2as().needs_refresh(DEFAULT_PFX2AS_CACHE_TTL) + /// Refresh RPKI data from provided records + /// + /// For loading from external sources (RTR, Cloudflare API, etc.), + /// fetch the data first, then pass it to this method. + pub fn refresh_rpki( + &self, + roas: &[RpkiRoaRecord], + aspas: &[RpkiAspaRecord], + roa_source: &str, + aspa_source: &str, + ) -> Result { + self.rpki().store(roas, aspas, roa_source, aspa_source)?; + Ok(RefreshResult::new( + roas.len() + aspas.len(), + format!("ROAs from: {}, ASPAs from: {}", roa_source, aspa_source), + )) } - /// Check if the Pfx2as cache needs refresh with custom TTL - pub fn needs_pfx2as_refresh_with_ttl(&self, ttl: chrono::Duration) -> bool { + // ========================================================================= + // Pfx2as Data Management + // ========================================================================= + + /// Check if Pfx2as data needs refresh + pub fn needs_pfx2as_refresh(&self, ttl: std::time::Duration) -> bool { self.pfx2as().needs_refresh(ttl) } + /// Refresh Pfx2as data from provided records + /// + /// For loading from external sources, fetch the data first, + /// then pass it to this method. + pub fn refresh_pfx2as( + &self, + records: &[Pfx2asDbRecord], + source: &str, + ) -> Result { + self.pfx2as().store(records, source)?; + Ok(RefreshResult::new(records.len(), source)) + } + + // ========================================================================= + // Deprecated aliases (for backward compatibility) + // ========================================================================= + + /// Deprecated: Use `needs_asinfo_refresh` instead + #[deprecated(since = "1.1.0", note = "Use needs_asinfo_refresh instead")] + pub fn needs_asinfo_bootstrap(&self) -> bool { + self.asinfo().is_empty() + } + + /// Deprecated: Use `refresh_asinfo` instead + #[deprecated(since = "1.1.0", note = "Use refresh_asinfo instead")] + pub fn bootstrap_asinfo(&self) -> Result { + self.refresh_asinfo() + } + + /// Deprecated: Use `refresh_as2rel` instead + #[deprecated(since = "1.1.0", note = "Use refresh_as2rel instead")] + pub fn update_as2rel(&self) -> Result { + self.refresh_as2rel() + } + + /// Deprecated: Use `refresh_as2rel_from` instead + #[deprecated(since = "1.1.0", note = "Use refresh_as2rel_from instead")] + pub fn update_as2rel_from(&self, path: &str) -> Result { + self.refresh_as2rel_from(path) + } + /// Get metadata value from the database pub fn get_meta(&self, key: &str) -> Result> { let schema = SchemaManager::new(&self.db.conn); @@ -223,13 +324,6 @@ mod tests { assert!(db.as2rel().is_empty()); } - #[test] - fn test_needs_bootstrap() { - let db = MonocleDatabase::open_in_memory().unwrap(); - - assert!(db.needs_as2rel_update()); - } - #[test] fn test_meta_operations() { let db = MonocleDatabase::open_in_memory().unwrap(); @@ -343,18 +437,6 @@ mod tests { assert_eq!(stats.invalid, 0); } - #[test] - fn test_needs_refresh_flags() { - let db = MonocleDatabase::open_in_memory().unwrap(); - - // Empty databases should need refresh - assert!(db.needs_asinfo_bootstrap()); - assert!(db.needs_asinfo_refresh()); - assert!(db.needs_as2rel_update()); - assert!(db.needs_rpki_refresh()); - assert!(db.needs_pfx2as_refresh()); - } - #[test] fn test_connection_accessible() { let db = MonocleDatabase::open_in_memory().unwrap(); diff --git a/src/database/monocle/pfx2as.rs b/src/database/monocle/pfx2as.rs index 4bb9003..7767f5d 100644 --- a/src/database/monocle/pfx2as.rs +++ b/src/database/monocle/pfx2as.rs @@ -17,19 +17,19 @@ //! - **Covered prefixes**: Find all prefixes covered by the query prefix use anyhow::{anyhow, Result}; -use chrono::{DateTime, Duration, Utc}; +use chrono::{DateTime, Utc}; use rusqlite::{params, Connection}; use serde::{Deserialize, Serialize}; use std::collections::HashSet; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::time::Duration; use tracing::info; -/// Default TTL for Pfx2as cache (24 hours) -pub const DEFAULT_PFX2AS_CACHE_TTL: Duration = Duration::hours(24); +/// Default TTL for Pfx2as cache (7 days) +pub const DEFAULT_PFX2AS_CACHE_TTL: Duration = Duration::from_secs(7 * 24 * 60 * 60); /// Pfx2as record for database storage -#[derive(Debug, Clone, Serialize, Deserialize)] -#[cfg_attr(feature = "display", derive(tabled::Tabled))] +#[derive(Debug, Clone, Serialize, Deserialize, tabled::Tabled)] pub struct Pfx2asDbRecord { /// IP prefix string (e.g., "1.1.1.0/24") pub prefix: String, @@ -187,7 +187,7 @@ impl<'a> Pfx2asRepository<'a> { } /// Check if the cache needs refresh based on TTL - pub fn needs_refresh(&self, ttl: Duration) -> bool { + pub fn needs_refresh(&self, ttl: std::time::Duration) -> bool { if !self.tables_exist() || self.is_empty() { return true; } @@ -195,7 +195,7 @@ impl<'a> Pfx2asRepository<'a> { match self.get_metadata() { Ok(Some(meta)) => { let age = Utc::now().signed_duration_since(meta.updated_at); - age > ttl + age.num_seconds() >= ttl.as_secs() as i64 } _ => true, } @@ -936,7 +936,7 @@ mod tests { assert!(!repo.needs_refresh(DEFAULT_PFX2AS_CACHE_TTL)); // With 0 TTL, should need refresh - assert!(repo.needs_refresh(Duration::zero())); + assert!(repo.needs_refresh(Duration::ZERO)); } #[test] diff --git a/src/database/monocle/rpki.rs b/src/database/monocle/rpki.rs index d150718..dac501d 100644 --- a/src/database/monocle/rpki.rs +++ b/src/database/monocle/rpki.rs @@ -17,14 +17,15 @@ //! - **NotFound**: No covering ROA exists for the prefix use anyhow::{anyhow, Result}; -use chrono::{DateTime, Duration, Utc}; +use chrono::{DateTime, Utc}; use rusqlite::{params, Connection}; use serde::{Deserialize, Serialize}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::time::Duration; use tracing::info; -/// Default TTL for RPKI cache (24 hours) -pub const DEFAULT_RPKI_CACHE_TTL: Duration = Duration::hours(24); +/// Default TTL for RPKI cache (7 days) +pub const DEFAULT_RPKI_CACHE_TTL: Duration = Duration::from_secs(7 * 24 * 60 * 60); /// RPKI validation state #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -49,8 +50,7 @@ impl std::fmt::Display for RpkiValidationState { } /// Detailed validation result -#[derive(Debug, Clone, Serialize, Deserialize)] -#[cfg_attr(feature = "display", derive(tabled::Tabled))] +#[derive(Debug, Clone, Serialize, Deserialize, tabled::Tabled)] pub struct RpkiValidationResult { pub prefix: String, pub asn: u32, @@ -59,8 +59,7 @@ pub struct RpkiValidationResult { } /// ROA record for database storage -#[derive(Debug, Clone, Serialize, Deserialize)] -#[cfg_attr(feature = "display", derive(tabled::Tabled))] +#[derive(Debug, Clone, Serialize, Deserialize, tabled::Tabled)] pub struct RpkiRoaRecord { pub prefix: String, pub max_length: u8, @@ -257,7 +256,7 @@ impl<'a> RpkiRepository<'a> { } /// Check if the cache needs refresh (expired or empty) - pub fn needs_refresh(&self, ttl: Duration) -> bool { + pub fn needs_refresh(&self, ttl: std::time::Duration) -> bool { if !self.tables_exist() || self.is_empty() { return true; } @@ -265,7 +264,8 @@ impl<'a> RpkiRepository<'a> { match self.get_metadata() { Ok(Some(meta)) => { let now = Utc::now(); - now.signed_duration_since(meta.updated_at) > ttl + let age = now.signed_duration_since(meta.updated_at); + age.num_seconds() >= ttl.as_secs() as i64 } _ => true, } @@ -1244,7 +1244,7 @@ mod tests { assert!(!repo.needs_refresh(DEFAULT_RPKI_CACHE_TTL)); // With zero TTL, should need refresh - assert!(repo.needs_refresh(Duration::zero())); + assert!(repo.needs_refresh(Duration::ZERO)); } #[test] diff --git a/src/database/session/mod.rs b/src/database/session/mod.rs index f9423ca..d69b267 100644 --- a/src/database/session/mod.rs +++ b/src/database/session/mod.rs @@ -10,11 +10,11 @@ //! //! # Feature Requirements //! -//! The `MsgStore` type requires the `lens-bgpkit` feature because it depends +//! The `MsgStore` type requires the `lib` feature because it depends //! on `bgpkit_parser::BgpElem` for storing BGP elements. -#[cfg(feature = "lens-bgpkit")] +#[cfg(feature = "lib")] mod msg_store; -#[cfg(feature = "lens-bgpkit")] +#[cfg(feature = "lib")] pub use msg_store::MsgStore; diff --git a/src/lens/README.md b/src/lens/README.md index e6ec203..cf99ee0 100644 --- a/src/lens/README.md +++ b/src/lens/README.md @@ -19,57 +19,54 @@ lens/ ├── README.md # This document ├── utils.rs # OutputFormat + formatting helpers │ -├── time/ # Time parsing / formatting (lens-core) +├── time/ # Time parsing / formatting │ └── mod.rs │ -├── country.rs # Country lookup (lens-bgpkit) -├── ip/ # IP information lookups (lens-bgpkit) +├── country/ # Country lookup (lib) +├── ip/ # IP information lookups │ └── mod.rs -├── parse/ # MRT parsing + progress callbacks (lens-bgpkit) +├── parse/ # MRT parsing + progress callbacks │ └── mod.rs -├── search/ # Search across public MRT files (lens-bgpkit) +├── search/ # Search across public MRT files │ ├── mod.rs │ └── query_builder.rs -├── rpki/ # RPKI operations (lens-bgpkit) +├── rpki/ # RPKI operations │ ├── mod.rs │ └── commons.rs -├── pfx2as/ # Prefix→AS mapping types (lens-bgpkit) +├── pfx2as/ # Prefix→AS mapping types │ └── mod.rs -├── as2rel/ # AS relationship lookups (lens-bgpkit) +├── as2rel/ # AS relationship lookups │ ├── mod.rs │ ├── args.rs │ └── types.rs │ -└── inspect/ # Unified AS/prefix inspection (lens-full) +└── inspect/ # Unified AS/prefix inspection ├── mod.rs # InspectLens implementation └── types.rs # Result types, section selection ``` --- -## Feature Tiers +## Feature Requirements -Lenses are organized by feature requirements: - -| Feature | Lenses | Key Dependencies | -|---------|--------|------------------| -| `lens-core` | `TimeLens` | chrono, dateparser | -| `lens-bgpkit` | `CountryLens`, `IpLens`, `ParseLens`, `SearchLens`, `RpkiLens`, `Pfx2asLens`, `As2relLens` | bgpkit-*, rayon, tabled | -| `lens-full` | `InspectLens` | All above | - -Library users can select minimal features: +All lenses are available with the `lib` feature, which is the default for library usage: ```toml -# Time parsing only -monocle = { version = "1.0", default-features = false, features = ["lens-core"] } - -# BGP operations without CLI -monocle = { version = "1.0", default-features = false, features = ["lens-bgpkit"] } - -# All lenses including InspectLens -monocle = { version = "1.0", default-features = false, features = ["lens-full"] } +# Library usage (all lenses + database) +monocle = { version = "1.0", default-features = false, features = ["lib"] } ``` +The `lib` feature includes: +- `TimeLens` - Time parsing and formatting +- `CountryLens` - Country code/name lookup +- `IpLens` - IP information lookup +- `ParseLens` - MRT file parsing +- `SearchLens` - BGP message search across MRT files +- `RpkiLens` - RPKI validation +- `Pfx2asLens` - Prefix-to-AS mapping +- `As2relLens` - AS relationship lookups +- `InspectLens` - Unified AS/prefix inspection + --- ## Design Philosophy @@ -176,7 +173,7 @@ These lenses do not require a persistent database reference: > Note: code below is intentionally example-focused; check the module docs / rustdoc for exact function signatures where needed. -### TimeLens (lens-core) +### TimeLens ```rust,ignore use monocle::lens::time::{TimeLens, TimeParseArgs}; @@ -193,7 +190,7 @@ let out = OutputFormat::Table.format(&results); println!("{}", out); ``` -### InspectLens (lens-full, database-backed) +### InspectLens (database-backed) ```rust,ignore use monocle::database::MonocleDatabase; @@ -219,7 +216,7 @@ for r in results { } ``` -### As2relLens (lens-bgpkit, database-backed) +### As2relLens (database-backed) ```rust,ignore use monocle::database::MonocleDatabase; @@ -240,7 +237,7 @@ let results = lens.search(&args)?; println!("{}", OutputFormat::Table.format(&results)); ``` -### SearchLens with progress (lens-bgpkit) +### SearchLens with progress ```rust,ignore use monocle::lens::search::{SearchLens, SearchFilters, SearchProgress}; @@ -277,10 +274,10 @@ For a detailed contributor walkthrough, see `DEVELOPMENT.md`. In short: - `Result` (output) - `Lens` (operations) 3. Add feature gate in `src/lens/mod.rs`: - ```rust - #[cfg(feature = "lens-bgpkit")] // or appropriate feature - pub mod newlens; - ``` + ```rust + #[cfg(feature = "lib")] + pub mod newlens; + ``` 4. Wire into (optional): - CLI command module under `src/bin/commands/` - WebSocket handler under `src/server/handlers/` diff --git a/src/lens/as2rel/mod.rs b/src/lens/as2rel/mod.rs index be7b604..ef252ef 100644 --- a/src/lens/as2rel/mod.rs +++ b/src/lens/as2rel/mod.rs @@ -18,6 +18,10 @@ pub use crate::lens::utils::{truncate_name, DEFAULT_NAME_MAX_LEN}; use crate::database::{MonocleDatabase, BGPKIT_AS2REL_URL}; use anyhow::Result; use serde_json::json; +use std::time::Duration; + +/// Default TTL for AS2Rel cache (7 days) +pub const DEFAULT_AS2REL_CACHE_TTL: Duration = Duration::from_secs(7 * 24 * 60 * 60); /// AS2Rel lens for querying AS-level relationships /// @@ -28,12 +32,22 @@ use serde_json::json; /// - Formatting results for output pub struct As2relLens<'a> { db: &'a MonocleDatabase, + /// TTL for cache staleness check + ttl: Duration, } impl<'a> As2relLens<'a> { - /// Create a new AS2Rel lens + /// Create a new AS2Rel lens with the default 7-day TTL pub fn new(db: &'a MonocleDatabase) -> Self { - Self { db } + Self { + db, + ttl: DEFAULT_AS2REL_CACHE_TTL, + } + } + + /// Create a new AS2Rel lens with a custom TTL + pub fn with_ttl(db: &'a MonocleDatabase, ttl: Duration) -> Self { + Self { db, ttl } } /// Check if data is available @@ -43,7 +57,7 @@ impl<'a> As2relLens<'a> { /// Check if data needs to be updated pub fn needs_update(&self) -> bool { - self.db.needs_as2rel_update() + self.db.needs_as2rel_refresh(self.ttl) } /// Check why the data needs update, if at all @@ -59,8 +73,8 @@ impl<'a> As2relLens<'a> { return Some(RefreshReason::Empty); } - // Check if outdated (uses should_update which checks 7-day TTL) - if self.db.needs_as2rel_update() { + // Check if outdated (uses configurable TTL) + if self.db.needs_as2rel_refresh(self.ttl) { return Some(RefreshReason::Outdated); } @@ -69,12 +83,12 @@ impl<'a> As2relLens<'a> { /// Update AS2Rel data from the default URL pub fn update(&self) -> Result { - self.db.update_as2rel() + self.db.refresh_as2rel() } /// Update AS2Rel data from a custom path pub fn update_from(&self, path: &str) -> Result { - self.db.update_as2rel_from(path) + self.db.refresh_as2rel_from(path) } /// Get the maximum peers count (for percentage calculation) @@ -381,61 +395,35 @@ impl<'a> As2relLens<'a> { serde_json::to_string_pretty(&output).unwrap_or_default() } As2relOutputFormat::Pretty => { - #[cfg(feature = "display")] - { - use tabled::settings::Style; - use tabled::Table; - if show_name { - let results_with_name: Vec<_> = results - .iter() - .cloned() - .map(|r| r.with_name(truncate_names)) - .collect(); - Table::new(&results_with_name) - .with(Style::rounded()) - .to_string() - } else { - Table::new(results).with(Style::rounded()).to_string() - } - } - #[cfg(not(feature = "display"))] - { - // Fall back to JSON when display feature is not enabled - self.format_results( - results, - &As2relOutputFormat::Json, - show_name, - truncate_names, - ) + use tabled::settings::Style; + use tabled::Table; + if show_name { + let results_with_name: Vec<_> = results + .iter() + .cloned() + .map(|r| r.with_name(truncate_names)) + .collect(); + Table::new(&results_with_name) + .with(Style::rounded()) + .to_string() + } else { + Table::new(results).with(Style::rounded()).to_string() } } As2relOutputFormat::Markdown => { - #[cfg(feature = "display")] - { - use tabled::settings::Style; - use tabled::Table; - if show_name { - let results_with_name: Vec<_> = results - .iter() - .cloned() - .map(|r| r.with_name(truncate_names)) - .collect(); - Table::new(&results_with_name) - .with(Style::markdown()) - .to_string() - } else { - Table::new(results).with(Style::markdown()).to_string() - } - } - #[cfg(not(feature = "display"))] - { - // Fall back to JSON when display feature is not enabled - self.format_results( - results, - &As2relOutputFormat::Json, - show_name, - truncate_names, - ) + use tabled::settings::Style; + use tabled::Table; + if show_name { + let results_with_name: Vec<_> = results + .iter() + .cloned() + .map(|r| r.with_name(truncate_names)) + .collect(); + Table::new(&results_with_name) + .with(Style::markdown()) + .to_string() + } else { + Table::new(results).with(Style::markdown()).to_string() } } } diff --git a/src/lens/as2rel/types.rs b/src/lens/as2rel/types.rs index c2e1d4d..0e44631 100644 --- a/src/lens/as2rel/types.rs +++ b/src/lens/as2rel/types.rs @@ -29,16 +29,15 @@ pub enum As2relOutputFormat { } /// Search result for AS relationships -#[derive(Debug, Clone, Serialize, Deserialize)] -#[cfg_attr(feature = "display", derive(tabled::Tabled))] +#[derive(Debug, Clone, Serialize, Deserialize, tabled::Tabled)] pub struct As2relSearchResult { pub asn1: u32, pub asn2: u32, - #[cfg_attr(feature = "display", tabled(skip))] + #[tabled(skip)] pub asn2_name: Option, /// Percentage of peers that see the connection (formatted string) pub connected: String, - #[cfg_attr(feature = "display", tabled(skip))] + #[tabled(skip)] pub connected_pct: f32, /// Percentage that see peer relationship pub peer: String, @@ -49,8 +48,7 @@ pub struct As2relSearchResult { } /// Search result with name displayed (for table output) -#[derive(Debug, Clone, Serialize, Deserialize)] -#[cfg_attr(feature = "display", derive(tabled::Tabled))] +#[derive(Debug, Clone, Serialize, Deserialize, tabled::Tabled)] pub struct As2relSearchResultWithName { pub asn1: u32, pub asn2: u32, diff --git a/src/lens/country.rs b/src/lens/country/mod.rs similarity index 92% rename from src/lens/country.rs rename to src/lens/country/mod.rs index 55ee582..f17c139 100644 --- a/src/lens/country.rs +++ b/src/lens/country/mod.rs @@ -5,7 +5,7 @@ //! //! # Feature Requirements //! -//! This module requires the `lens-bgpkit` feature. +//! This module requires the `lib` feature. //! //! # Example //! @@ -64,8 +64,7 @@ impl CountryData { // ============================================================================= /// A country entry with code and name -#[derive(Debug, Clone, Serialize, Deserialize)] -#[cfg_attr(feature = "display", derive(tabled::Tabled))] +#[derive(Debug, Clone, Serialize, Deserialize, tabled::Tabled)] pub struct CountryEntry { /// ISO 3166-1 alpha-2 country code pub code: String, @@ -273,30 +272,14 @@ impl CountryLens { match format { CountryOutputFormat::Table => { - #[cfg(feature = "display")] - { - use tabled::settings::Style; - use tabled::Table; - Table::new(results).with(Style::rounded()).to_string() - } - #[cfg(not(feature = "display"))] - { - // Fall back to Simple format - self.format_results(results, &CountryOutputFormat::Simple) - } + use tabled::settings::Style; + use tabled::Table; + Table::new(results).with(Style::rounded()).to_string() } CountryOutputFormat::Markdown => { - #[cfg(feature = "display")] - { - use tabled::settings::Style; - use tabled::Table; - Table::new(results).with(Style::markdown()).to_string() - } - #[cfg(not(feature = "display"))] - { - // Fall back to Simple format - self.format_results(results, &CountryOutputFormat::Simple) - } + use tabled::settings::Style; + use tabled::Table; + Table::new(results).with(Style::markdown()).to_string() } CountryOutputFormat::Json => serde_json::to_string_pretty(results).unwrap_or_default(), CountryOutputFormat::Simple => results diff --git a/src/lens/inspect/mod.rs b/src/lens/inspect/mod.rs index f006e55..35ca914 100644 --- a/src/lens/inspect/mod.rs +++ b/src/lens/inspect/mod.rs @@ -22,16 +22,14 @@ pub mod types; pub use types::*; -use crate::database::{ - AsinfoCoreRecord, AsinfoFullRecord, AsinfoStoreCounts, MonocleDatabase, - DEFAULT_PFX2AS_CACHE_TTL, DEFAULT_RPKI_CACHE_TTL, -}; +use crate::config::MonocleConfig; +use crate::database::{AsinfoCoreRecord, AsinfoFullRecord, AsinfoStoreCounts, MonocleDatabase}; use crate::lens::country::CountryLens; use anyhow::{anyhow, Result}; use serde::Serialize; use std::collections::{HashMap, HashSet}; use std::net::IpAddr; -use std::time::Instant; +use std::time::{Duration, Instant}; use tabled::settings::Style; use tabled::{Table, Tabled}; use tracing::info; @@ -108,18 +106,37 @@ impl DataRefreshSummary { /// - Getting AS connectivity information pub struct InspectLens<'a> { db: &'a MonocleDatabase, + config: &'a MonocleConfig, country_lookup: CountryLens, } impl<'a> InspectLens<'a> { /// Create a new Inspect lens - pub fn new(db: &'a MonocleDatabase) -> Self { + pub fn new(db: &'a MonocleDatabase, config: &'a MonocleConfig) -> Self { Self { db, + config, country_lookup: CountryLens::new(), } } + // Helper methods for TTL access + fn asinfo_ttl(&self) -> Duration { + self.config.asinfo_cache_ttl() + } + + fn as2rel_ttl(&self) -> Duration { + self.config.as2rel_cache_ttl() + } + + fn rpki_ttl(&self) -> Duration { + self.config.rpki_cache_ttl() + } + + fn pfx2as_ttl(&self) -> Duration { + self.config.pfx2as_cache_ttl() + } + // ========================================================================= // Status Methods // ========================================================================= @@ -131,12 +148,12 @@ impl<'a> InspectLens<'a> { /// Check if data needs to be bootstrapped pub fn needs_bootstrap(&self) -> bool { - self.db.needs_asinfo_bootstrap() + self.db.needs_asinfo_refresh(self.asinfo_ttl()) } /// Check if data needs refresh pub fn needs_refresh(&self) -> bool { - self.db.needs_asinfo_refresh() + self.db.needs_asinfo_refresh(self.asinfo_ttl()) } // ========================================================================= @@ -146,13 +163,13 @@ impl<'a> InspectLens<'a> { /// Bootstrap ASInfo data from the default URL pub fn bootstrap(&self) -> Result { info!("Bootstrapping ASInfo data..."); - self.db.bootstrap_asinfo() + self.db.refresh_asinfo() } /// Refresh ASInfo data (same as bootstrap, but logs differently) pub fn refresh(&self) -> Result { info!("Refreshing ASInfo data..."); - self.db.bootstrap_asinfo() + self.db.refresh_asinfo() } /// Ensure all required data sources are available, refreshing if needed @@ -172,7 +189,7 @@ impl<'a> InspectLens<'a> { if self.db.asinfo().is_empty() { eprintln!("[monocle] Loading ASInfo data (AS names, organizations, PeeringDB)..."); info!("ASInfo data is empty, bootstrapping..."); - match self.db.bootstrap_asinfo() { + match self.db.refresh_asinfo() { Ok(counts) => { summary.add( "asinfo", @@ -193,10 +210,10 @@ impl<'a> InspectLens<'a> { ); } } - } else if self.db.needs_asinfo_refresh() { + } else if self.db.needs_asinfo_refresh(self.asinfo_ttl()) { eprintln!("[monocle] Refreshing ASInfo data (AS names, organizations, PeeringDB)..."); info!("ASInfo data is stale, refreshing..."); - match self.db.bootstrap_asinfo() { + match self.db.refresh_asinfo() { Ok(counts) => { summary.add( "asinfo", @@ -223,7 +240,7 @@ impl<'a> InspectLens<'a> { if self.db.as2rel().is_empty() { eprintln!("[monocle] Loading AS2Rel data (AS relationships)..."); info!("AS2Rel data is empty, loading..."); - match self.db.update_as2rel() { + match self.db.refresh_as2rel() { Ok(count) => { summary.add( "as2rel", @@ -241,10 +258,10 @@ impl<'a> InspectLens<'a> { ); } } - } else if self.db.needs_as2rel_update() { + } else if self.db.needs_as2rel_refresh(self.as2rel_ttl()) { eprintln!("[monocle] Refreshing AS2Rel data (AS relationships)..."); info!("AS2Rel data is stale, refreshing..."); - match self.db.update_as2rel() { + match self.db.refresh_as2rel() { Ok(count) => { summary.add( "as2rel", @@ -281,7 +298,7 @@ impl<'a> InspectLens<'a> { summary.add("rpki", false, format!("Failed to load RPKI: {}", e), None); } } - } else if self.db.rpki().needs_refresh(DEFAULT_RPKI_CACHE_TTL) { + } else if self.db.rpki().needs_refresh(self.rpki_ttl()) { eprintln!("[monocle] Refreshing RPKI data (ROAs, ASPA)..."); info!("RPKI data is stale, refreshing..."); match self.refresh_rpki_from_commons() { @@ -326,7 +343,7 @@ impl<'a> InspectLens<'a> { ); } } - } else if self.db.pfx2as().needs_refresh(DEFAULT_PFX2AS_CACHE_TTL) { + } else if self.db.pfx2as().needs_refresh(self.pfx2as_ttl()) { eprintln!("[monocle] Refreshing Pfx2as data (prefix-to-AS mappings)..."); info!("Pfx2as data is stale, refreshing..."); match self.refresh_pfx2as() { @@ -366,7 +383,7 @@ impl<'a> InspectLens<'a> { if sections.contains(&InspectDataSection::Basic) { if self.db.asinfo().is_empty() { eprintln!("[monocle] Loading ASInfo data (AS names, organizations, PeeringDB)..."); - match self.db.bootstrap_asinfo() { + match self.db.refresh_asinfo() { Ok(counts) => { summary.add( "asinfo", @@ -387,11 +404,11 @@ impl<'a> InspectLens<'a> { ); } } - } else if self.db.needs_asinfo_refresh() { + } else if self.db.needs_asinfo_refresh(self.asinfo_ttl()) { eprintln!( "[monocle] Refreshing ASInfo data (AS names, organizations, PeeringDB)..." ); - match self.db.bootstrap_asinfo() { + match self.db.refresh_asinfo() { Ok(counts) => { summary.add( "asinfo", @@ -419,7 +436,7 @@ impl<'a> InspectLens<'a> { if sections.contains(&InspectDataSection::Connectivity) { if self.db.as2rel().is_empty() { eprintln!("[monocle] Loading AS2Rel data (AS relationships)..."); - match self.db.update_as2rel() { + match self.db.refresh_as2rel() { Ok(count) => { summary.add( "as2rel", @@ -437,9 +454,9 @@ impl<'a> InspectLens<'a> { ); } } - } else if self.db.needs_as2rel_update() { + } else if self.db.needs_as2rel_refresh(self.as2rel_ttl()) { eprintln!("[monocle] Refreshing AS2Rel data (AS relationships)..."); - match self.db.update_as2rel() { + match self.db.refresh_as2rel() { Ok(count) => { summary.add( "as2rel", @@ -477,7 +494,7 @@ impl<'a> InspectLens<'a> { summary.add("rpki", false, format!("Failed to load RPKI: {}", e), None); } } - } else if self.db.rpki().needs_refresh(DEFAULT_RPKI_CACHE_TTL) { + } else if self.db.rpki().needs_refresh(self.rpki_ttl()) { eprintln!("[monocle] Refreshing RPKI data (ROAs, ASPA)..."); match self.refresh_rpki_from_commons() { Ok(count) => { @@ -522,7 +539,7 @@ impl<'a> InspectLens<'a> { ); } } - } else if self.db.pfx2as().needs_refresh(DEFAULT_PFX2AS_CACHE_TTL) { + } else if self.db.pfx2as().needs_refresh(self.pfx2as_ttl()) { eprintln!("[monocle] Refreshing Pfx2as data (prefix-to-AS mappings)..."); match self.refresh_pfx2as() { Ok(count) => { @@ -2289,10 +2306,16 @@ impl<'a> InspectLens<'a> { mod tests { use super::*; + /// Helper to create a lens with default config for tests + fn create_test_lens<'a>(db: &'a MonocleDatabase, config: &'a MonocleConfig) -> InspectLens<'a> { + InspectLens::new(db, config) + } + #[test] fn test_detect_query_type_asn() { let db = MonocleDatabase::open_in_memory().unwrap(); - let lens = InspectLens::new(&db); + let config = MonocleConfig::default(); + let lens = create_test_lens(&db, &config); assert_eq!(lens.detect_query_type("13335"), InspectQueryType::Asn); assert_eq!(lens.detect_query_type("AS13335"), InspectQueryType::Asn); @@ -2303,7 +2326,8 @@ mod tests { #[test] fn test_detect_query_type_prefix() { let db = MonocleDatabase::open_in_memory().unwrap(); - let lens = InspectLens::new(&db); + let config = MonocleConfig::default(); + let lens = create_test_lens(&db, &config); assert_eq!( lens.detect_query_type("1.1.1.0/24"), @@ -2320,7 +2344,8 @@ mod tests { #[test] fn test_detect_query_type_name() { let db = MonocleDatabase::open_in_memory().unwrap(); - let lens = InspectLens::new(&db); + let config = MonocleConfig::default(); + let lens = create_test_lens(&db, &config); assert_eq!(lens.detect_query_type("cloudflare"), InspectQueryType::Name); assert_eq!(lens.detect_query_type("Google LLC"), InspectQueryType::Name); @@ -2330,7 +2355,8 @@ mod tests { #[test] fn test_parse_asn() { let db = MonocleDatabase::open_in_memory().unwrap(); - let lens = InspectLens::new(&db); + let config = MonocleConfig::default(); + let lens = create_test_lens(&db, &config); assert_eq!(lens.parse_asn("13335").unwrap(), 13335); assert_eq!(lens.parse_asn("AS13335").unwrap(), 13335); @@ -2341,7 +2367,8 @@ mod tests { #[test] fn test_normalize_prefix() { let db = MonocleDatabase::open_in_memory().unwrap(); - let lens = InspectLens::new(&db); + let config = MonocleConfig::default(); + let lens = create_test_lens(&db, &config); assert_eq!(lens.normalize_prefix("1.1.1.0/24").unwrap(), "1.1.1.0/24"); assert_eq!(lens.normalize_prefix("1.1.1.1").unwrap(), "1.1.1.1/32"); diff --git a/src/lens/mod.rs b/src/lens/mod.rs index c9427eb..fdf5388 100644 --- a/src/lens/mod.rs +++ b/src/lens/mod.rs @@ -6,19 +6,24 @@ //! //! # Feature Requirements //! -//! Lenses are organized by the features they require: +//! All lenses require the `lib` feature to be enabled. //! -//! | Lens | Feature Required | Dependencies | -//! |------|-----------------|--------------| -//! | `TimeLens` | `lens-core` | chrono, dateparser | -//! | `CountryLens` | `lens-bgpkit` | bgpkit-commons | -//! | `IpLens` | `lens-bgpkit` | ureq, radar-rs | -//! | `ParseLens` | `lens-bgpkit` | bgpkit-parser | -//! | `SearchLens` | `lens-bgpkit` | bgpkit-broker, bgpkit-parser, rayon | -//! | `RpkiLens` | `lens-bgpkit` | bgpkit-commons | -//! | `Pfx2asLens` | `lens-bgpkit` | bgpkit-commons, oneio | -//! | `As2relLens` | `lens-bgpkit` | (database only) | -//! | `InspectLens` | `lens-full` | All above | +//! **Quick Guide:** +//! - Need the CLI binary? Use `cli` feature (includes everything) +//! - Need WebSocket server without CLI? Use `server` feature (includes lib) +//! - Need only library/data access? Use `lib` feature (this module) +//! +//! | Lens | Description | Dependencies | +//! |------|-------------|--------------| +//! | `TimeLens` | Time parsing and formatting | chrono, dateparser | +//! | `CountryLens` | Country code/name lookup | bgpkit-commons | +//! | `IpLens` | IP information lookup | ureq, radar-rs | +//! | `ParseLens` | MRT file parsing | bgpkit-parser | +//! | `SearchLens` | BGP message search | bgpkit-broker, bgpkit-parser, rayon | +//! | `RpkiLens` | RPKI validation and data | bgpkit-commons | +//! | `Pfx2asLens` | Prefix-to-ASN mapping | bgpkit-commons, oneio | +//! | `As2relLens` | AS-level relationships | database | +//! | `InspectLens` | Unified AS/prefix lookup | All above | //! //! # Architecture //! @@ -36,68 +41,60 @@ //! specific lens module you need: //! //! ```rust,ignore -//! // Time parsing (lens-core) +//! // Time parsing //! use monocle::lens::time::{TimeLens, TimeParseArgs, TimeOutputFormat}; //! -//! // RPKI validation (lens-bgpkit) +//! // RPKI validation //! use monocle::lens::rpki::{RpkiLens, RpkiValidationArgs, RpkiListArgs, RpkiRoaEntry}; //! -//! // IP information (lens-bgpkit) +//! // IP information //! use monocle::lens::ip::{IpLens, IpLookupArgs, IpInfo}; //! -//! // Unified AS/prefix inspection (lens-full) +//! // Unified AS/prefix inspection //! use monocle::lens::inspect::{InspectLens, InspectQueryOptions}; //! ``` // ============================================================================= -// Utility module (always available when any lens feature is enabled) +// Utility module (always available when lib feature is enabled) // ============================================================================= pub mod utils; // ============================================================================= -// Core lenses (lens-core feature) +// All lenses (require lib feature) // ============================================================================= // TimeLens - time parsing and formatting -#[cfg(feature = "lens-core")] +#[cfg(feature = "lib")] pub mod time; -// ============================================================================= -// BGPKIT lenses (lens-bgpkit feature) -// ============================================================================= - // CountryLens - country code/name lookup using bgpkit-commons -#[cfg(feature = "lens-bgpkit")] +#[cfg(feature = "lib")] pub mod country; // IpLens - IP information lookup -#[cfg(feature = "lens-bgpkit")] +#[cfg(feature = "lib")] pub mod ip; // ParseLens - MRT file parsing with bgpkit-parser -#[cfg(feature = "lens-bgpkit")] +#[cfg(feature = "lib")] pub mod parse; // SearchLens - BGP message search across MRT files -#[cfg(feature = "lens-bgpkit")] +#[cfg(feature = "lib")] pub mod search; // RpkiLens - RPKI validation and data -#[cfg(feature = "lens-bgpkit")] +#[cfg(feature = "lib")] pub mod rpki; // Pfx2asLens - prefix-to-ASN mapping -#[cfg(feature = "lens-bgpkit")] +#[cfg(feature = "lib")] pub mod pfx2as; -// As2relLens - AS-level relationships (uses database, but grouped with bgpkit for convenience) -#[cfg(feature = "lens-bgpkit")] +// As2relLens - AS-level relationships +#[cfg(feature = "lib")] pub mod as2rel; -// ============================================================================= -// Full lenses (lens-full feature) -// ============================================================================= - // InspectLens - unified AS and prefix information lookup -#[cfg(feature = "lens-full")] +#[cfg(feature = "lib")] pub mod inspect; diff --git a/src/lens/pfx2as/mod.rs b/src/lens/pfx2as/mod.rs index 37102f0..b123f0e 100644 --- a/src/lens/pfx2as/mod.rs +++ b/src/lens/pfx2as/mod.rs @@ -368,17 +368,20 @@ impl<'a> Pfx2asLens<'a> { } /// Check if the cache needs refresh (empty or expired) - pub fn needs_refresh(&self) -> Result { - Ok(self - .db - .pfx2as() - .needs_refresh(crate::database::DEFAULT_PFX2AS_CACHE_TTL)) + /// + /// Uses the provided TTL to determine if the cache is stale. + pub fn needs_refresh(&self, ttl: std::time::Duration) -> Result { + Ok(self.db.pfx2as().needs_refresh(ttl)) } /// Check why the cache needs refresh, if at all /// /// Returns `Some(RefreshReason)` if refresh is needed, `None` if data is current. - pub fn refresh_reason(&self) -> Result> { + /// Uses the provided TTL to determine if the cache is stale. + pub fn refresh_reason( + &self, + ttl: std::time::Duration, + ) -> Result> { use crate::lens::utils::RefreshReason; let pfx2as = self.db.pfx2as(); @@ -389,7 +392,7 @@ impl<'a> Pfx2asLens<'a> { } // Check if outdated - if pfx2as.needs_refresh(crate::database::DEFAULT_PFX2AS_CACHE_TTL) { + if pfx2as.needs_refresh(ttl) { return Ok(Some(RefreshReason::Outdated)); } diff --git a/src/lens/rpki/mod.rs b/src/lens/rpki/mod.rs index 2383c28..2703afb 100644 --- a/src/lens/rpki/mod.rs +++ b/src/lens/rpki/mod.rs @@ -383,17 +383,20 @@ impl<'a> RpkiLens<'a> { } /// Check if the cache needs refresh (empty or expired) - pub fn needs_refresh(&self) -> Result { - Ok(self - .db - .rpki() - .needs_refresh(crate::database::DEFAULT_RPKI_CACHE_TTL)) + /// + /// Uses the provided TTL to determine if the cache is stale. + pub fn needs_refresh(&self, ttl: std::time::Duration) -> Result { + Ok(self.db.rpki().needs_refresh(ttl)) } /// Check why the cache needs refresh, if at all /// /// Returns `Some(RefreshReason)` if refresh is needed, `None` if data is current. - pub fn refresh_reason(&self) -> Result> { + /// Uses the provided TTL to determine if the cache is stale. + pub fn refresh_reason( + &self, + ttl: std::time::Duration, + ) -> Result> { use crate::lens::utils::RefreshReason; let rpki = self.db.rpki(); @@ -404,7 +407,7 @@ impl<'a> RpkiLens<'a> { } // Check if outdated - if rpki.needs_refresh(crate::database::DEFAULT_RPKI_CACHE_TTL) { + if rpki.needs_refresh(ttl) { return Ok(Some(RefreshReason::Outdated)); } diff --git a/src/lens/time/mod.rs b/src/lens/time/mod.rs index 60a2ca3..1c206e1 100644 --- a/src/lens/time/mod.rs +++ b/src/lens/time/mod.rs @@ -93,8 +93,7 @@ where // ============================================================================= /// Represents a parsed BGP time with multiple format representations -#[derive(Debug, Clone, Serialize, Deserialize)] -#[cfg_attr(feature = "display", derive(tabled::Tabled))] +#[derive(Debug, Clone, Serialize, Deserialize, tabled::Tabled)] pub struct TimeBgpTime { /// Unix timestamp in seconds pub unix: i64, @@ -281,17 +280,9 @@ impl TimeLens { pub fn format_results(&self, results: &[TimeBgpTime], format: &TimeOutputFormat) -> String { match format { TimeOutputFormat::Table => { - #[cfg(feature = "display")] - { - use tabled::settings::Style; - use tabled::Table; - Table::new(results).with(Style::rounded()).to_string() - } - #[cfg(not(feature = "display"))] - { - // Fall back to JSON when display feature is not enabled - serde_json::to_string_pretty(results).unwrap_or_default() - } + use tabled::settings::Style; + use tabled::Table; + Table::new(results).with(Style::rounded()).to_string() } TimeOutputFormat::Rfc3339 => results .iter() diff --git a/src/lens/utils.rs b/src/lens/utils.rs index 1b0e24a..1e2df0d 100644 --- a/src/lens/utils.rs +++ b/src/lens/utils.rs @@ -386,6 +386,123 @@ impl std::fmt::Display for RefreshReason { } } +// ============================================================================= +// Cache TTL Configuration +// ============================================================================= + +use std::time::Duration; + +/// Default TTL for all caches (7 days in seconds) +pub const DEFAULT_CACHE_TTL_SECS: u64 = 7 * 24 * 60 * 60; + +/// Configuration for cache time-to-live (TTL) settings +/// +/// This struct encapsulates TTL values for all data sources, providing a clean +/// interface for configuring cache expiration across the application. +/// +/// # Example +/// +/// ```rust +/// use monocle::lens::utils::CacheTtlConfig; +/// use std::time::Duration; +/// +/// // Use default 7-day TTL for all sources +/// let config = CacheTtlConfig::default(); +/// +/// // Custom TTLs +/// let config = CacheTtlConfig::new() +/// .with_asinfo(Duration::from_secs(3600)) // 1 hour +/// .with_rpki(Duration::from_secs(86400)); // 1 day +/// ``` +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CacheTtlConfig { + /// TTL for ASInfo data (AS names, organizations, PeeringDB) + pub asinfo: Duration, + /// TTL for AS2Rel data (AS relationships) + pub as2rel: Duration, + /// TTL for RPKI data (ROAs, ASPAs) + pub rpki: Duration, + /// TTL for Pfx2as data (prefix-to-ASN mappings) + pub pfx2as: Duration, +} + +impl Default for CacheTtlConfig { + fn default() -> Self { + let default_ttl = Duration::from_secs(DEFAULT_CACHE_TTL_SECS); + Self { + asinfo: default_ttl, + as2rel: default_ttl, + rpki: default_ttl, + pfx2as: default_ttl, + } + } +} + +impl CacheTtlConfig { + /// Create a new CacheTtlConfig with default 7-day TTL for all sources + pub fn new() -> Self { + Self::default() + } + + /// Set all TTLs to the same value + pub fn with_all(mut self, ttl: Duration) -> Self { + self.asinfo = ttl; + self.as2rel = ttl; + self.rpki = ttl; + self.pfx2as = ttl; + self + } + + /// Set the ASInfo TTL + pub fn with_asinfo(mut self, ttl: Duration) -> Self { + self.asinfo = ttl; + self + } + + /// Set the AS2Rel TTL + pub fn with_as2rel(mut self, ttl: Duration) -> Self { + self.as2rel = ttl; + self + } + + /// Set the RPKI TTL + pub fn with_rpki(mut self, ttl: Duration) -> Self { + self.rpki = ttl; + self + } + + /// Set the Pfx2as TTL + pub fn with_pfx2as(mut self, ttl: Duration) -> Self { + self.pfx2as = ttl; + self + } + + /// Create from individual Duration values + pub fn from_durations( + asinfo: Duration, + as2rel: Duration, + rpki: Duration, + pfx2as: Duration, + ) -> Self { + Self { + asinfo, + as2rel, + rpki, + pfx2as, + } + } + + /// Create from seconds values + pub fn from_secs(asinfo: u64, as2rel: u64, rpki: u64, pfx2as: u64) -> Self { + Self { + asinfo: Duration::from_secs(asinfo), + as2rel: Duration::from_secs(as2rel), + rpki: Duration::from_secs(rpki), + pfx2as: Duration::from_secs(pfx2as), + } + } +} + // ============================================================================= // Output Format and Display Utilities // ============================================================================= diff --git a/src/lib.rs b/src/lib.rs index 9a68427..d6b4e2b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,33 +9,24 @@ //! //! # Feature Flags //! -//! Monocle uses a layered feature system to minimize dependencies based on your needs: +//! Monocle uses a simplified feature system with three options: //! -//! | Feature | Description | Key Dependencies | -//! |---------|-------------|------------------| -//! | `database` | SQLite database operations only | `rusqlite` | -//! | `lens-core` | Standalone lenses (TimeLens, OutputFormat) | `chrono-humanize`, `dateparser` | -//! | `lens-bgpkit` | BGP-related lenses (Parse, Search, RPKI, Country) | `bgpkit-*`, `rayon` | -//! | `lens-full` | All lenses including InspectLens | All above | -//! | `display` | Table formatting with `tabled` | `tabled` | -//! | `cli` | Full CLI binary with server support | All above + `clap`, `axum` | +//! | Feature | Description | Implies | +//! |---------|-------------|---------| +//! | `lib` | Complete library (database + all lenses + display) | - | +//! | `server` | WebSocket server for programmatic API access | `lib` | +//! | `cli` | Full CLI binary with all functionality | `lib`, `server` | //! //! ## Choosing Features //! //! ```toml -//! # Minimal - just database operations -//! monocle = { version = "1.0", default-features = false, features = ["database"] } +//! # Library-only - all lenses and database operations +//! monocle = { version = "1.0", default-features = false, features = ["lib"] } //! -//! # Standalone utilities (time parsing, output formatting) -//! monocle = { version = "1.0", default-features = false, features = ["lens-core"] } +//! # Library + WebSocket server +//! monocle = { version = "1.0", default-features = false, features = ["server"] } //! -//! # BGP operations without CLI overhead -//! monocle = { version = "1.0", default-features = false, features = ["lens-bgpkit"] } -//! -//! # Full lens functionality with display support -//! monocle = { version = "1.0", default-features = false, features = ["lens-full", "display"] } -//! -//! # Default (CLI binary) +//! # Default (full CLI binary) //! monocle = "1.0" //! ``` //! @@ -43,27 +34,29 @@ //! //! The library is organized into the following modules: //! -//! - **[`database`]**: Database functionality (requires `database` feature) +//! - **[`database`]**: Database functionality (requires `lib` feature) //! - `core`: SQLite connection management and schema definitions //! - `session`: One-time storage (e.g., search results) //! - `monocle`: Main monocle database (ASInfo, AS2Rel, RPKI, Pfx2as) //! -//! - **[`lens`]**: High-level business logic (feature-gated) -//! - `time`: Time parsing and formatting (requires `lens-core`) -//! - `country`: Country code/name lookup (requires `lens-bgpkit`) -//! - `ip`: IP information lookup (requires `lens-bgpkit`) -//! - `parse`: MRT file parsing (requires `lens-bgpkit`) -//! - `search`: BGP message search (requires `lens-bgpkit`) -//! - `rpki`: RPKI validation and data (requires `lens-bgpkit`) -//! - `pfx2as`: Prefix-to-ASN mapping (requires `lens-bgpkit`) -//! - `as2rel`: AS-level relationships (requires `lens-bgpkit`) -//! - `inspect`: Unified AS/prefix lookup (requires `lens-full`) +//! - **[`lens`]**: High-level business logic (requires `lib` feature) +//! - `time`: Time parsing and formatting +//! - `country`: Country code/name lookup +//! - `ip`: IP information lookup +//! - `parse`: MRT file parsing +//! - `search`: BGP message search +//! - `rpki`: RPKI validation and data +//! - `pfx2as`: Prefix-to-ASN mapping +//! - `as2rel`: AS-level relationships +//! - `inspect`: Unified AS/prefix lookup +//! +//! - **[`server`]**: WebSocket API server (requires `server` feature) //! //! - **[`config`]**: Configuration management (always available) //! //! # Quick Start Examples //! -//! ## Database Operations (feature = "database") +//! ## Database Operations (feature = "lib") //! //! ```rust,ignore //! use monocle::database::MonocleDatabase; @@ -72,7 +65,9 @@ //! let db = MonocleDatabase::open_in_dir("~/.monocle")?; //! //! // Check if AS2Rel data needs update -//! if db.needs_as2rel_update() { +//! use std::time::Duration; +//! let ttl = Duration::from_secs(24 * 60 * 60); // 24 hours +//! if db.needs_as2rel_refresh(ttl) { //! let count = db.update_as2rel()?; //! println!("Loaded {} relationships", count); //! } @@ -84,7 +79,7 @@ //! } //! ``` //! -//! ## Time Parsing (feature = "lens-core") +//! ## Time Parsing (feature = "lib") //! //! ```rust,ignore //! use monocle::lens::time::{TimeLens, TimeParseArgs}; @@ -101,7 +96,7 @@ //! } //! ``` //! -//! ## RPKI Validation (feature = "lens-bgpkit") +//! ## RPKI Validation (feature = "lib") //! //! ```rust,ignore //! use monocle::database::MonocleDatabase; @@ -120,7 +115,7 @@ //! println!("{}: {}", result.state, result.reason); //! ``` //! -//! ## MRT Parsing with Progress (feature = "lens-bgpkit") +//! ## MRT Parsing with Progress (feature = "lib") //! //! ```rust,ignore //! use monocle::lens::parse::{ParseLens, ParseFilters, ParseProgress}; @@ -141,7 +136,7 @@ //! let elems = lens.parse_with_progress(&filters, "path/to/file.mrt", Some(callback))?; //! ``` //! -//! ## Unified Inspection (feature = "lens-full") +//! ## Unified Inspection (feature = "lib") //! //! ```rust,ignore //! use monocle::database::MonocleDatabase; @@ -163,15 +158,15 @@ //! ``` pub mod config; -#[cfg(feature = "database")] +#[cfg(feature = "lib")] pub mod database; -// Lens module - feature gated -#[cfg(any(feature = "lens-core", feature = "lens-bgpkit", feature = "lens-full"))] +// Lens module - requires lib feature +#[cfg(feature = "lib")] pub mod lens; -// Server module - requires CLI feature -#[cfg(feature = "cli")] +// Server module - requires server feature +#[cfg(feature = "server")] pub mod server; // ============================================================================= @@ -181,9 +176,9 @@ pub mod server; pub use config::MonocleConfig; // Shared database info types (used by config and database commands) -#[cfg(feature = "database")] +#[cfg(feature = "lib")] pub use config::get_data_source_info; -#[cfg(feature = "database")] +#[cfg(feature = "lib")] pub use config::get_sqlite_info; pub use config::{ format_size, get_cache_settings, CacheSettings, DataSource, DataSourceInfo, DataSourceStatus, @@ -194,22 +189,22 @@ pub use config::{ // Database Module - Re-export all public types // ============================================================================= -#[cfg(feature = "database")] +#[cfg(feature = "lib")] pub use database::*; // ============================================================================= // Lens Module - Feature-gated exports // ============================================================================= -// Output format utilities (lens-core) -#[cfg(any(feature = "lens-core", feature = "lens-bgpkit", feature = "lens-full"))] +// Output format utilities (lib feature) +#[cfg(feature = "lib")] pub use lens::utils::OutputFormat; // ============================================================================= -// Server Module (WebSocket API) - requires "cli" feature +// Server Module (WebSocket API) - requires "server" feature // ============================================================================= -#[cfg(feature = "cli")] +#[cfg(feature = "server")] pub use server::{ create_router, start_server, Dispatcher, OperationRegistry, Router, ServerConfig, ServerState, WsContext, WsError, WsMethod, WsRequest, WsResult, WsSink, diff --git a/src/server/handler.rs b/src/server/handler.rs index e8e0180..118c59b 100644 --- a/src/server/handler.rs +++ b/src/server/handler.rs @@ -15,6 +15,8 @@ use std::sync::Arc; // Context // ============================================================================= +use crate::config::MonocleConfig; + /// WebSocket context providing access to shared resources /// /// This context is passed to all handlers and provides access to: @@ -24,27 +26,25 @@ use std::sync::Arc; /// - Rate limiting state #[derive(Clone)] pub struct WsContext { - /// Path to the monocle data directory - pub data_dir: String, + /// Monocle configuration (includes data_dir and cache TTLs) + pub config: MonocleConfig, } impl WsContext { - /// Create a new WebSocket context - /// - /// Note: transport policy (message size, timeouts, concurrency limits) is owned by `ServerConfig` - /// and enforced in the connection loop / dispatcher layer. - pub fn new(data_dir: String) -> Self { - Self { data_dir } + /// Create a new WebSocket context from MonocleConfig + pub fn from_config(config: MonocleConfig) -> Self { + Self { config } + } + + /// Get the data directory path + pub fn data_dir(&self) -> &str { + &self.config.data_dir } } impl Default for WsContext { fn default() -> Self { - let home_dir = dirs::home_dir() - .map(|h| h.to_string_lossy().to_string()) - .unwrap_or_else(|| ".".to_string()); - - Self::new(format!("{}/.monocle", home_dir)) + Self::from_config(MonocleConfig::default()) } } @@ -251,13 +251,14 @@ mod tests { #[test] fn test_ws_context_default() { let ctx = WsContext::default(); - assert!(ctx.data_dir.contains(".monocle")); + assert!(ctx.data_dir().contains(".monocle")); } #[test] - fn test_ws_context_new() { - let ctx = WsContext::new("/tmp/test".to_string()); - assert_eq!(ctx.data_dir, "/tmp/test"); + fn test_ws_context_from_config() { + let config = MonocleConfig::default(); + let ctx = WsContext::from_config(config.clone()); + assert_eq!(ctx.data_dir(), &config.data_dir); } #[test] diff --git a/src/server/handlers/as2rel.rs b/src/server/handlers/as2rel.rs index c4e0ad0..d165ca1 100644 --- a/src/server/handlers/as2rel.rs +++ b/src/server/handlers/as2rel.rs @@ -150,7 +150,7 @@ impl WsMethod for As2relSearchHandler { // Do all DB work synchronously first, then await only for sending the response. let response = { // Open the database - let db = MonocleDatabase::open_in_dir(&ctx.data_dir).map_err(|e| { + let db = MonocleDatabase::open_in_dir(ctx.data_dir()).map_err(|e| { WsError::operation_failed(format!("Failed to open database: {}", e)) })?; @@ -252,7 +252,7 @@ impl WsMethod for As2relRelationshipHandler { // We do all DB work synchronously first, then await only for sending the response. // Open the database - let db = MonocleDatabase::open_in_dir(&ctx.data_dir) + let db = MonocleDatabase::open_in_dir(ctx.data_dir()) .map_err(|e| WsError::operation_failed(format!("Failed to open database: {}", e)))?; // Run search (DB-first) without any `.await` diff --git a/src/server/handlers/database.rs b/src/server/handlers/database.rs index b9ce7de..971f839 100644 --- a/src/server/handlers/database.rs +++ b/src/server/handlers/database.rs @@ -99,7 +99,7 @@ impl WsMethod for DatabaseStatusHandler { sink: WsOpSink, ) -> WsResult<()> { // Build paths - let sqlite_path = format!("{}/monocle.db", ctx.data_dir); + let sqlite_path = format!("{}/monocle.db", ctx.data_dir()); let sqlite_exists = Path::new(&sqlite_path).exists(); // Get SQLite size if exists @@ -112,7 +112,7 @@ impl WsMethod for DatabaseStatusHandler { // Open database to get counts let (as2rel_count, rpki_roa_count, pfx2as_count, as2rel_status, rpki_status, pfx2as_status) = if sqlite_exists { - match MonocleDatabase::open_in_dir(&ctx.data_dir) { + match MonocleDatabase::open_in_dir(ctx.data_dir()) { Ok(db) => { let as2rel = db.as2rel().count().unwrap_or(0); let rpki_roa = db.rpki().roa_count().unwrap_or(0); @@ -269,12 +269,12 @@ impl WsMethod for DatabaseRefreshHandler { let _force = params.force.unwrap_or(false); // Open the database - let db = MonocleDatabase::open_in_dir(&ctx.data_dir) + let db = MonocleDatabase::open_in_dir(ctx.data_dir()) .map_err(|e| WsError::operation_failed(format!("Failed to open database: {}", e)))?; let (message, count) = match source.as_str() { "as2rel" => { - let count = db.update_as2rel().map_err(|e| { + let count = db.refresh_as2rel().map_err(|e| { WsError::operation_failed(format!("AS2Rel refresh failed: {}", e)) })?; ( @@ -331,7 +331,7 @@ impl WsMethod for DatabaseRefreshHandler { let mut messages = Vec::new(); // AS2Rel - match db.update_as2rel() { + match db.refresh_as2rel() { Ok(count) => messages.push(format!("AS2Rel: {} entries", count)), Err(e) => messages.push(format!("AS2Rel: failed - {}", e)), } diff --git a/src/server/handlers/inspect.rs b/src/server/handlers/inspect.rs index 3fb368f..3d1578e 100644 --- a/src/server/handlers/inspect.rs +++ b/src/server/handlers/inspect.rs @@ -142,10 +142,10 @@ impl WsMethod for InspectQueryHandler { // Do all DB work in a block before any awaits to avoid Send issues let (refresh_summary, result): (DataRefreshSummary, InspectResult) = { - let db = MonocleDatabase::open_in_dir(&ctx.data_dir) + let db = MonocleDatabase::open_in_dir(ctx.data_dir()) .map_err(|e| WsError::internal(format!("Failed to open database: {}", e)))?; - let lens = InspectLens::new(&db); + let lens = InspectLens::new(&db, &ctx.config); // Ensure data is available, refreshing if needed let refresh_summary = lens @@ -298,10 +298,10 @@ impl WsMethod for InspectRefreshHandler { ) -> WsResult<()> { // Do all DB work in a block before any awaits let summary: DataRefreshSummary = { - let db = MonocleDatabase::open_in_dir(&ctx.data_dir) + let db = MonocleDatabase::open_in_dir(ctx.data_dir()) .map_err(|e| WsError::internal(format!("Failed to open database: {}", e)))?; - let lens = InspectLens::new(&db); + let lens = InspectLens::new(&db, &ctx.config); // Perform refresh lens.ensure_data_available() diff --git a/src/server/handlers/pfx2as.rs b/src/server/handlers/pfx2as.rs index 4a181c9..634f840 100644 --- a/src/server/handlers/pfx2as.rs +++ b/src/server/handlers/pfx2as.rs @@ -95,7 +95,7 @@ impl WsMethod for Pfx2asLookupHandler { // Do all DB work before any await to avoid Send issues with rusqlite::Connection let response: Pfx2asLookupResponse = { - let db = MonocleDatabase::open_in_dir(&ctx.data_dir) + let db = MonocleDatabase::open_in_dir(ctx.data_dir()) .map_err(|e| WsError::internal(format!("Failed to open database: {}", e)))?; let repo = db.pfx2as(); diff --git a/src/server/handlers/rpki.rs b/src/server/handlers/rpki.rs index ac90737..47695b1 100644 --- a/src/server/handlers/rpki.rs +++ b/src/server/handlers/rpki.rs @@ -117,7 +117,7 @@ impl WsMethod for RpkiValidateHandler { // values across an `.await`. Do all DB work first, then await only to send. let response = { // Open the database - let db = MonocleDatabase::open_in_dir(&ctx.data_dir).map_err(|e| { + let db = MonocleDatabase::open_in_dir(ctx.data_dir()).map_err(|e| { WsError::operation_failed(format!("Failed to open database: {}", e)) })?; @@ -289,7 +289,7 @@ impl WsMethod for RpkiRoasHandler { // Do all DB work before any `.await`. let response = { // DB-first: query local database only. - let db = MonocleDatabase::open_in_dir(&ctx.data_dir).map_err(|e| { + let db = MonocleDatabase::open_in_dir(ctx.data_dir()).map_err(|e| { WsError::operation_failed(format!("Failed to open database: {}", e)) })?; @@ -445,7 +445,7 @@ impl WsMethod for RpkiAspasHandler { // Do all DB work before any `.await`. let response = { // DB-first: query local database only. - let db = MonocleDatabase::open_in_dir(&ctx.data_dir).map_err(|e| { + let db = MonocleDatabase::open_in_dir(ctx.data_dir()).map_err(|e| { WsError::operation_failed(format!("Failed to open database: {}", e)) })?; diff --git a/src/server/mod.rs b/src/server/mod.rs index 5b65c6e..8c16c5e 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -27,16 +27,18 @@ //! //! ```rust,ignore //! use monocle::server::{create_router, WsContext, ServerConfig}; +//! use monocle::config::MonocleConfig; //! //! // Create the router with all handlers registered //! let router = create_router(); //! -//! // Create context -//! let context = WsContext::new("~/.monocle".to_string()); +//! // Create context from config +//! let config = MonocleConfig::new(&None)?; +//! let context = WsContext::from_config(config); //! //! // Start the server -//! let config = ServerConfig::default(); -//! start_server(router, context, config).await?; +//! let server_config = ServerConfig::default(); +//! start_server(router, context, server_config).await?; //! ``` pub mod handler;