add geometry repair to pluto

.github/workflows/ingest_single.yml

@@ -7,7 +7,7 @@ on:
       dataset:
         description: "Name of the dataset (required)"
         required: true
-        default: dcp_mappluto
+        default: dcp_mappluto_wi
       latest:
         type: boolean
         description: "Tag this version as latest (optional)"

.github/workflows/pluto_build.yml

@@ -83,18 +83,9 @@ jobs:
       run: python -m dcpy.lifecycle.builds.load load --recipe-path ${{ inputs.recipe_file
         }}.lock.yml
 
-    - name: Load Local Data
-      run: ./01_load_local_csvs.sh
-
-    - name: building ...
+    - name: Build
       run: ./02_build.sh
 
-    - name: apply corrections
-      run: ./03_corrections.sh
-
-    - name: Archive
-      run: ./04_archive.sh
-
     - name: QAQC
       run: ./05_qaqc.sh
 

products/pluto/pluto_build/02_build.sh

@@ -3,6 +3,7 @@ source ./bash/config.sh
 set_error_traps
 
 echo "Starting to build PLUTO ..."
+run_sql_file sql/_create.sql
 run_sql_file sql/preprocessing.sql
 run_sql_file sql/create_pts.sql
 run_sql_file sql/create_rpad_geo.sql
@@ -108,4 +109,44 @@ run_sql_file sql/plutomapid_1.sql
 run_sql_file sql/plutomapid_2.sql
 run_sql_file sql/shorelineclip.sql
 
+run_sql_file sql/corr_create.sql
+
+echo "Applying corrections to PLUTO"
+run_sql_file sql/corr_lotarea.sql
+run_sql_file sql/corr_template.sql -v FIELD=yearbuilt
+run_sql_file sql/corr_template.sql -v FIELD=ownername
+run_sql_file sql/corr_ownername_punctuation.sql
+run_sql_file sql/corr_template.sql -v FIELD=cd
+run_sql_file sql/corr_template.sql -v FIELD=numfloors
+run_sql_file sql/corr_template.sql -v FIELD=numbldgs
+run_sql_file sql/corr_template.sql -v FIELD=unitsres
+run_sql_file sql/corr_template.sql -v FIELD=unitstotal
+run_sql_file sql/corr_inwoodrezoning.sql
+run_sql_file sql/corr_template.sql -v FIELD=bct2020
+run_sql_file sql/corr_template.sql -v FIELD=address
+run_sql_file sql/remove_unitlots.sql
+
+echo "Creating export tables"
+run_sql_file sql/export.sql
+
+run_sql_file \
+  sql/export_mappluto_shp.sql\
+  -v TABLE='mappluto'\
+  -v GEOM='clipped_2263'
+
+run_sql_file \
+  sql/export_mappluto_shp.sql\
+  -v TABLE='mappluto_unclipped'\
+  -v GEOM='geom_2263'
+
+run_sql_file \
+  sql/export_mappluto_gdb.sql\
+  -v TABLE='mappluto_gdb'\
+  -v GEOM='clipped_2263'
+
+run_sql_file \
+  sql/export_mappluto_gdb.sql\
+  -v TABLE='mappluto_unclipped_gdb'\
+  -v GEOM='geom_2263'
+
 echo 'Done'

products/pluto/tests/assert_all_geometry_is_valid.sql

@@ -0,0 +1,20 @@
+{{ 
+    config(
+        tags = ['de_check', 'minor', 'major'], 
+        meta = {
+            'description': '''
+				This test checks for any invalid lot geometries
+			''',
+            'next_steps': 'Contact DE and GIS to investigate'
+        }
+    ) 
+}}
+
+WITH export_pluto AS (
+    SELECT *
+    FROM {{ source('build_sources', 'export_pluto') }}
+)
+
+SELECT *
+FROM export_pluto
+WHERE zonedist NOT IN (SELECT zonedist FROM far_lookup)