chk

fvankrieken · fvankrieken · commit 27cd146eda33 · 2025-11-21T13:07:28.000-05:00
diff --git a/products/cscl/models/intermediate/adjacent_polygons/int__segment_atomicpolygons.sql b/products/cscl/models/intermediate/adjacent_polygons/int__segment_atomicpolygons.sql
@@ -10,6 +10,7 @@ WITH segment_offsets AS (
 
 SELECT
     so.lionkey,
+    so.globalid,
     so.segmentid,
     so.boroughcode AS segment_borocode,
     left_poly.atomicid AS left_atomicid,
diff --git a/products/cscl/models/intermediate/adjacent_polygons/int__segment_offsets.sql b/products/cscl/models/intermediate/adjacent_polygons/int__segment_offsets.sql
@@ -8,6 +8,7 @@ WITH segments AS (
 
 SELECT
     lionkey,
+    globalid,
     segmentid,
     boroughcode,
     feature_type,
diff --git a/products/cscl/models/intermediate/nodes/int__segments_to_nodes.sql b/products/cscl/models/intermediate/nodes/int__segments_to_nodes.sql
@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'table',
     indexes=[
-      {'columns': ['segmentid']},
+      {'columns': ['lionkey']},
       {'columns': ['nodeid']},
     ]
 ) }}
diff --git a/products/cscl/models/intermediate/nodes/int__segments_with_nodes.sql b/products/cscl/models/intermediate/nodes/int__segments_with_nodes.sql
@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'table',
     indexes=[
-      {'columns': ['segmentid']},
+      {'columns': ['lionkey']},
     ]
 ) }}
 
diff --git a/products/cscl/models/intermediate/segments/int__primary_segments.sql b/products/cscl/models/intermediate/segments/int__primary_segments.sql
@@ -112,4 +112,3 @@ segment_attributes AS (
 )
 
 SELECT * FROM segment_attributes
-WHERE face_code IS NOT NULL -- TODO error report for this and maybe refactor to get this in a more logical place
diff --git a/products/cscl/models/intermediate/segments/int__protosegments.sql b/products/cscl/models/intermediate/segments/int__protosegments.sql
@@ -18,7 +18,7 @@ SELECT
     CASE
         WHEN feature_type_codes.source_feature_class = 'centerline' THEN facecode.street_facecode
         ELSE facecode.feature_facecode
-    END AS face_code, -- TODO error report when null
+    END AS face_code,
     CASE
         WHEN feature_type_codes.source_feature_class <> 'nonstreetfeatures' THEN proto.alt_segment_seqnum
         ELSE seqnum.segment_seqnum
@@ -67,7 +67,7 @@ SELECT
     proto.source_table,
     proto.globalid
 FROM proto
-INNER JOIN primary_segments ON proto.segmentid = primary_segments.segmentid -- TODO error report for non-matches
+LEFT JOIN primary_segments ON proto.segmentid = primary_segments.segmentid
 LEFT JOIN facecode ON proto.b7sc = facecode.b7sc
 LEFT JOIN seqnum ON proto.globalid = seqnum.globalid
 LEFT JOIN feature_type_codes ON proto.feature_type_code IS NOT DISTINCT FROM feature_type_codes.code -- NULL -> centerline
diff --git a/products/cscl/models/intermediate/segments/int__segments.sql b/products/cscl/models/intermediate/segments/int__segments.sql
@@ -13,9 +13,11 @@ WITH segments AS (
     SELECT
         {{ dbt_utils.star(ref('int__primary_segments')) }}
     FROM {{ ref("int__protosegments") }}
+    WHERE geom IS NOT NULL -- proxy for joined to a segment
 )
 SELECT
     CONCAT(boroughcode, face_code, segment_seqnum) AS lionkey,
     *
 FROM segments
+WHERE face_code IS NOT NULL
 ORDER BY lionkey
diff --git a/products/cscl/models/log_files/log.sql b/products/cscl/models/log_files/log.sql
@@ -0,0 +1,5 @@
+{% set log_tables = dbt_utils.get_relations_by_pattern(
+    table_pattern='log__%'
+) %}
+
+{{ dbt_utils.union_relations(relations=log_tables) }}
diff --git a/products/cscl/models/log_files/log__centerline_or_proto_seglogstatus_mismatch.sql b/products/cscl/models/log_files/log__centerline_or_proto_seglogstatus_mismatch.sql
@@ -0,0 +1,32 @@
+WITH lion AS (
+    SELECT * FROM {{ ref("int__lion") }}
+),
+centerline AS (
+    SELECT * FROM {{ ref("stg__centerline") }}
+),
+proto AS (
+    SELECT * FROM {{ ref("stg__altsegmentdata_proto") }}
+),
+joined AS (
+    SELECT
+        lion.lionkey,
+        lion.globalid,
+        lion.segmentid,
+        lion.segment_locational_status,
+        CASE
+            WHEN lion.source_table = 'centerline' THEN centerline.seglocstatus
+            WHEN lion.source_table = 'altsegmentdata' THEN proto.seglocstatus
+        END AS source_segment_locational_status,
+        lion.source_table
+    FROM lion
+    LEFT JOIN centerline ON lion.source_table = 'centerline' AND lion.segmentid = centerline.segmentid
+    LEFT JOIN proto ON lion.source_table = 'altsegmentdata' AND lion.ogc_fid = proto.ogc_fid
+)
+SELECT
+    'seglocstatus mismatch' AS error,
+    globalid,
+    source_table,
+    'segmentid' AS record_id_type,
+    segmentid AS record_id
+FROM joined
+WHERE segment_locational_status IS DISTINCT FROM source_segment_locational_status
diff --git a/products/cscl/models/log_files/log__protosegment_orphans.sql b/products/cscl/models/log_files/log__protosegment_orphans.sql
@@ -0,0 +1,8 @@
+SELECT DISTINCT
+    'protosegment without geometry-modeled segment' AS error,
+    globalid
+    source_table,
+    'segmentid' AS record_id_type,
+    segmentid AS record_id
+FROM {{ ref('int__protosegments') }}
+WHERE geom IS NULL
diff --git a/products/cscl/models/log_files/log__segment_ap_boro_mismatch.sql b/products/cscl/models/log_files/log__segment_ap_boro_mismatch.sql
@@ -0,0 +1,8 @@
+SELECT 
+    'neither joined atomic polygon matches segment''s borocode' AS error,
+    globalid,
+    source_table,
+    'segmentid' AS record_id_type,
+    segmentid AS record_id
+FROM {{ ref('int__segment_atomicpolygons') }}
+WHERE left_borocode IS DISTINCT FROM segment_borocode AND right_borocode IS DISTINCT FROM segment_borocode
diff --git a/products/cscl/models/log_files/log__segment_missing_nypd.sql b/products/cscl/models/log_files/log__segment_missing_nypd.sql
@@ -0,0 +1,8 @@
+SELECT 
+    'segment joined to no nypd beat' AS error,
+    globalid,
+    source_table,
+    'segmentid' AS record_id_type,
+    segmentid AS record_id
+FROM {{ ref('int__lion') }}
+WHERE left_nypd_service_area IS NULL AND right_nypd_service_area IS NULL
diff --git a/products/cscl/models/log_files/log__segments_missing_aps.sql b/products/cscl/models/log_files/log__segments_missing_aps.sql
@@ -0,0 +1,8 @@
+SELECT 
+    'segment joined to no atomic polygon' AS error,
+    globalid,
+    source_table,
+    'segmentid' AS record_id_type,
+    segmentid AS record_id
+FROM {{ ref('int__lion') }}
+WHERE left_atomicid IS NULL AND right_atomicid IS NULL
diff --git a/products/cscl/models/log_files/log__segments_missing_facecode.sql b/products/cscl/models/log_files/log__segments_missing_facecode.sql
@@ -0,0 +1,18 @@
+WITH segments AS (
+    SELECT
+        {{ dbt_utils.star(ref('int__primary_segments')) }}
+    FROM {{ ref("int__primary_segments") }}
+    UNION ALL
+    SELECT
+        {{ dbt_utils.star(ref('int__primary_segments')) }}
+    FROM {{ ref("int__protosegments") }}
+)
+SELECT DISTINCT
+    'missing facecode' AS error,
+    globalid,
+    source_table,
+    'segmentid' AS record_id_type,
+    segmentid AS record_id
+FROM segments
+WHERE face_code IS NULL
+ORDER BY lionkey
diff --git a/products/cscl/models/log_files/log__segments_missing_nodes.sql b/products/cscl/models/log_files/log__segments_missing_nodes.sql
@@ -0,0 +1,8 @@
+SELECT 
+    'segment missing node' AS error,
+    globalid,
+    source_table,
+    'segmentid' AS record_id_type,
+    segmentid AS record_id
+FROM {{ ref('int__lion') }}
+WHERE from_nodeid IS NULL OR to_nodeid IS NULL

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`{{ config(`
`2`	`2`	`materialized = 'table',`
`3`	`3`	`indexes=[`
`4`		`- {'columns': ['segmentid']},`
	`4`	`+ {'columns': ['lionkey']},`
`5`	`5`	`{'columns': ['nodeid']},`
`6`	`6`	`]`
`7`	`7`	`) }}`
Original file line number	Diff line number	Diff line change
`@@ -112,4 +112,3 @@ segment_attributes AS (`
`112`	`112`	`)`
`113`	`113`
`114`	`114`	`SELECT * FROM segment_attributes`
`115`		`-WHERE face_code IS NOT NULL -- TODO error report for this and maybe refactor to get this in a more logical place`
Original file line number	Diff line number	Diff line change
`@@ -13,9 +13,11 @@ WITH segments AS (`
`13`	`13`	`SELECT`
`14`	`14`	`{{ dbt_utils.star(ref('int__primary_segments')) }}`
`15`	`15`	`FROM {{ ref("int__protosegments") }}`
	`16`	`+ WHERE geom IS NOT NULL -- proxy for joined to a segment`
`16`	`17`	`)`
`17`	`18`	`SELECT`
`18`	`19`	`CONCAT(boroughcode, face_code, segment_seqnum) AS lionkey,`
`19`	`20`	`*`
`20`	`21`	`FROM segments`
	`22`	`+WHERE face_code IS NOT NULL`
`21`	`23`	`ORDER BY lionkey`