Skip to content

Commit fa3f58d

Browse files
mjpitzmjpitz
authored
feat: bring up to date and plumb in a few components
1 parent a641058 commit fa3f58d

File tree

16 files changed

+1263
-14
lines changed

16 files changed

+1263
-14
lines changed

‎.context/index.md‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
---
2+
# Basic project information
3+
module-name: cognative
4+
description: |
5+
Traditional approaches to observability, operations, and business intelligence treat these concerns as separate and
6+
distinct. This often results in several, different technology stacks, often dramatically increasing the demand and
7+
requirements of your on-call staff.
8+
9+
Cognative deploys a turn-key solution that centralizes log, metric, trace, and wide-event data into a common store and
10+
exposes them for analysis behind a single pane of glass. Traditional approaches often require engineering effort to
11+
centralize data from numerous, distinct sources into a common location. This results in toil and requires further
12+
maintenance as the organization grows. Cognative removes this toil by centralizing the data FIRST.
13+
14+
# Technical stack
15+
technologies:
16+
- Docker
17+
- Kubernetes
18+
- Helm
19+
- SQL
20+
- Clickhouse
21+
- OpenTelemetry
22+
23+
# Development conventions
24+
conventions:
25+
- PNPM monorepo
26+
- SPDX license header
27+
- Conventional Commits
28+
- Prettier + ESLint
29+
---
30+
31+
## Overview
32+
33+
This document provides an overview of the ecosystem that exists within this repository. It captures important context,
34+
patterns, and memories that are useful for AI code agents to produce and manipulate code.
35+
36+
## Architecture Details
37+
38+
### Component Interactions

‎.context/schema/histogram_metrics.sql‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
-- Default Histogram metrics table DDL
2+
3+
CREATE TABLE IF NOT EXISTS otel_metrics_histogram (
4+
ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
5+
ResourceSchemaUrl String CODEC(ZSTD(1)),
6+
ScopeName String CODEC(ZSTD(1)),
7+
ScopeVersion String CODEC(ZSTD(1)),
8+
ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
9+
ScopeDroppedAttrCount UInt32 CODEC(ZSTD(1)),
10+
ScopeSchemaUrl String CODEC(ZSTD(1)),
11+
ServiceName LowCardinality(String) CODEC(ZSTD(1)),
12+
MetricName String CODEC(ZSTD(1)),
13+
MetricDescription String CODEC(ZSTD(1)),
14+
MetricUnit String CODEC(ZSTD(1)),
15+
Attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
16+
StartTimeUnix DateTime64(9) CODEC(Delta, ZSTD(1)),
17+
TimeUnix DateTime64(9) CODEC(Delta, ZSTD(1)),
18+
Count UInt64 CODEC(Delta, ZSTD(1)),
19+
Sum Float64 CODEC(ZSTD(1)),
20+
BucketCounts Array(UInt64) CODEC(ZSTD(1)),
21+
ExplicitBounds Array(Float64) CODEC(ZSTD(1)),
22+
Exemplars Nested (
23+
FilteredAttributes Map(LowCardinality(String), String),
24+
TimeUnix DateTime64(9),
25+
Value Float64,
26+
SpanId String,
27+
TraceId String
28+
) CODEC(ZSTD(1)),
29+
Flags UInt32 CODEC(ZSTD(1)),
30+
Min Float64 CODEC(ZSTD(1)),
31+
Max Float64 CODEC(ZSTD(1)),
32+
AggregationTemporality Int32 CODEC(ZSTD(1)),
33+
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
34+
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
35+
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
36+
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
37+
INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
38+
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
39+
) ENGINE = MergeTree()
40+
TTL toDateTime("TimeUnix") + toIntervalDay(180)
41+
PARTITION BY toDate(TimeUnix)
42+
ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
43+
SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;

‎.context/schema/logs.sql‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
-- Default Logs table DDL
2+
3+
CREATE TABLE IF NOT EXISTS otel_logs (
4+
Timestamp DateTime64(9) CODEC(Delta(8), ZSTD(1)),
5+
TimestampTime DateTime DEFAULT toDateTime(Timestamp),
6+
TraceId String CODEC(ZSTD(1)),
7+
SpanId String CODEC(ZSTD(1)),
8+
TraceFlags UInt8,
9+
SeverityText LowCardinality(String) CODEC(ZSTD(1)),
10+
SeverityNumber UInt8,
11+
ServiceName LowCardinality(String) CODEC(ZSTD(1)),
12+
Body String CODEC(ZSTD(1)),
13+
ResourceSchemaUrl LowCardinality(String) CODEC(ZSTD(1)),
14+
ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
15+
ScopeSchemaUrl LowCardinality(String) CODEC(ZSTD(1)),
16+
ScopeName String CODEC(ZSTD(1)),
17+
ScopeVersion LowCardinality(String) CODEC(ZSTD(1)),
18+
ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
19+
LogAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
20+
21+
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
22+
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
23+
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
24+
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
25+
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
26+
INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
27+
INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
28+
INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
29+
) ENGINE = MergeTree()
30+
PARTITION BY toDate(TimestampTime)
31+
PRIMARY KEY (ServiceName, TimestampTime)
32+
ORDER BY (ServiceName, TimestampTime, Timestamp)
33+
TTL TimestampTime + toIntervalDay(180)
34+
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;

‎.context/schema/sum_metrics.sql‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
-- Default Sum metrics table DDL
2+
3+
CREATE TABLE IF NOT EXISTS otel_metrics_sum (
4+
ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
5+
ResourceSchemaUrl String CODEC(ZSTD(1)),
6+
ScopeName String CODEC(ZSTD(1)),
7+
ScopeVersion String CODEC(ZSTD(1)),
8+
ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
9+
ScopeDroppedAttrCount UInt32 CODEC(ZSTD(1)),
10+
ScopeSchemaUrl String CODEC(ZSTD(1)),
11+
ServiceName LowCardinality(String) CODEC(ZSTD(1)),
12+
MetricName String CODEC(ZSTD(1)),
13+
MetricDescription String CODEC(ZSTD(1)),
14+
MetricUnit String CODEC(ZSTD(1)),
15+
Attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
16+
StartTimeUnix DateTime64(9) CODEC(Delta, ZSTD(1)),
17+
TimeUnix DateTime64(9) CODEC(Delta, ZSTD(1)),
18+
Value Float64 CODEC(ZSTD(1)),
19+
Flags UInt32 CODEC(ZSTD(1)),
20+
Exemplars Nested (
21+
FilteredAttributes Map(LowCardinality(String), String),
22+
TimeUnix DateTime64(9),
23+
Value Float64,
24+
SpanId String,
25+
TraceId String
26+
) CODEC(ZSTD(1)),
27+
AggregationTemporality Int32 CODEC(ZSTD(1)),
28+
IsMonotonic Boolean CODEC(Delta, ZSTD(1)),
29+
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
30+
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
31+
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
32+
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
33+
INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
34+
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
35+
) ENGINE = MergeTree()
36+
TTL toDateTime("TimeUnix") + toIntervalDay(180)
37+
PARTITION BY toDate(TimeUnix)
38+
ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
39+
SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;

‎.context/schema/summary_metrics.sql‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
-- Default Summary metrics DDL
2+
3+
CREATE TABLE IF NOT EXISTS otel_metrics_summary (
4+
ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
5+
ResourceSchemaUrl String CODEC(ZSTD(1)),
6+
ScopeName String CODEC(ZSTD(1)),
7+
ScopeVersion String CODEC(ZSTD(1)),
8+
ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
9+
ScopeDroppedAttrCount UInt32 CODEC(ZSTD(1)),
10+
ScopeSchemaUrl String CODEC(ZSTD(1)),
11+
ServiceName LowCardinality(String) CODEC(ZSTD(1)),
12+
MetricName String CODEC(ZSTD(1)),
13+
MetricDescription String CODEC(ZSTD(1)),
14+
MetricUnit String CODEC(ZSTD(1)),
15+
Attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
16+
StartTimeUnix DateTime64(9) CODEC(Delta, ZSTD(1)),
17+
TimeUnix DateTime64(9) CODEC(Delta, ZSTD(1)),
18+
Count UInt64 CODEC(Delta, ZSTD(1)),
19+
Sum Float64 CODEC(ZSTD(1)),
20+
ValueAtQuantiles Nested(
21+
Quantile Float64,
22+
Value Float64
23+
) CODEC(ZSTD(1)),
24+
Flags UInt32 CODEC(ZSTD(1)),
25+
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
26+
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
27+
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
28+
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
29+
INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
30+
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
31+
) ENGINE = MergeTree()
32+
TTL toDateTime("TimeUnix") + toIntervalDay(180)
33+
PARTITION BY toDate(TimeUnix)
34+
ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
35+
SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;

‎.context/schema/traces.sql‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
-- Default Trace table DDL
2+
3+
CREATE TABLE IF NOT EXISTS otel_traces (
4+
Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)),
5+
TraceId String CODEC(ZSTD(1)),
6+
SpanId String CODEC(ZSTD(1)),
7+
ParentSpanId String CODEC(ZSTD(1)),
8+
TraceState String CODEC(ZSTD(1)),
9+
SpanName LowCardinality(String) CODEC(ZSTD(1)),
10+
SpanKind LowCardinality(String) CODEC(ZSTD(1)),
11+
ServiceName LowCardinality(String) CODEC(ZSTD(1)),
12+
ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
13+
ScopeName String CODEC(ZSTD(1)),
14+
ScopeVersion String CODEC(ZSTD(1)),
15+
SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
16+
Duration UInt64 CODEC(ZSTD(1)),
17+
StatusCode LowCardinality(String) CODEC(ZSTD(1)),
18+
StatusMessage String CODEC(ZSTD(1)),
19+
Events Nested (
20+
Timestamp DateTime64(9),
21+
Name LowCardinality(String),
22+
Attributes Map(LowCardinality(String), String)
23+
) CODEC(ZSTD(1)),
24+
Links Nested (
25+
TraceId String,
26+
SpanId String,
27+
TraceState String,
28+
Attributes Map(LowCardinality(String), String)
29+
) CODEC(ZSTD(1)),
30+
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
31+
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
32+
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
33+
INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
34+
INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
35+
INDEX idx_duration Duration TYPE minmax GRANULARITY 1
36+
) ENGINE = MergeTree()
37+
PARTITION BY toDate(Timestamp)
38+
ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
39+
TTL toDate(Timestamp) + toIntervalDay(180)
40+
SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
41+
42+
43+
CREATE TABLE IF NOT EXISTS otel_traces_trace_id_ts (
44+
TraceId String CODEC(ZSTD(1)),
45+
Start DateTime CODEC(Delta, ZSTD(1)),
46+
End DateTime CODEC(Delta, ZSTD(1)),
47+
INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1
48+
) ENGINE = MergeTree()
49+
PARTITION BY toDate(Start)
50+
ORDER BY (TraceId, Start)
51+
TTL toDate(Start) + toIntervalDay(180)
52+
SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
53+
54+
55+
CREATE MATERIALIZED VIEW IF NOT EXISTS otel_traces_trace_id_ts_mv
56+
TO otel_traces_trace_id_ts
57+
AS SELECT
58+
TraceId,
59+
min(Timestamp) as Start,
60+
max(Timestamp) as End
61+
FROM otel_traces
62+
WHERE TraceId != ''
63+
GROUP BY TraceId;

0 commit comments

Comments
 (0)