diff --git a/.github/workflows/ingest_single.yml b/.github/workflows/ingest_single.yml index f4734d0f4d..c714bf4ff8 100644 --- a/.github/workflows/ingest_single.yml +++ b/.github/workflows/ingest_single.yml @@ -7,7 +7,7 @@ on: dataset: description: "Name of the dataset (required)" required: true - default: dcp_mappluto + default: dcp_mappluto_wi latest: type: boolean description: "Tag this version as latest (optional)" diff --git a/.github/workflows/pluto_build.yml b/.github/workflows/pluto_build.yml index 0b36ebd559..b50544103e 100644 --- a/.github/workflows/pluto_build.yml +++ b/.github/workflows/pluto_build.yml @@ -83,18 +83,9 @@ jobs: run: python -m dcpy.lifecycle.builds.load load --recipe-path ${{ inputs.recipe_file }}.lock.yml - - name: Load Local Data - run: ./01_load_local_csvs.sh - - - name: building ... + - name: Build run: ./02_build.sh - - name: apply corrections - run: ./03_corrections.sh - - - name: Archive - run: ./04_archive.sh - - name: QAQC run: ./05_qaqc.sh diff --git a/products/pluto/pluto_build/01_load_local_csvs.sh b/products/pluto/pluto_build/01_load_local_csvs.sh deleted file mode 100755 index 49e4ec0cb3..0000000000 --- a/products/pluto/pluto_build/01_load_local_csvs.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -source ./bash/config.sh -set_error_traps - -run_sql_file sql/_create.sql diff --git a/products/pluto/pluto_build/02_build.sh b/products/pluto/pluto_build/02_build.sh index 5657da5a6e..35bb8b217e 100755 --- a/products/pluto/pluto_build/02_build.sh +++ b/products/pluto/pluto_build/02_build.sh @@ -3,6 +3,7 @@ source ./bash/config.sh set_error_traps echo "Starting to build PLUTO ..." +run_sql_file sql/_create.sql run_sql_file sql/preprocessing.sql run_sql_file sql/create_pts.sql run_sql_file sql/create_rpad_geo.sql @@ -108,4 +109,44 @@ run_sql_file sql/plutomapid_1.sql run_sql_file sql/plutomapid_2.sql run_sql_file sql/shorelineclip.sql +run_sql_file sql/corr_create.sql + +echo "Applying corrections to PLUTO" +run_sql_file sql/corr_lotarea.sql +run_sql_file sql/corr_template.sql -v FIELD=yearbuilt +run_sql_file sql/corr_template.sql -v FIELD=ownername +run_sql_file sql/corr_ownername_punctuation.sql +run_sql_file sql/corr_template.sql -v FIELD=cd +run_sql_file sql/corr_template.sql -v FIELD=numfloors +run_sql_file sql/corr_template.sql -v FIELD=numbldgs +run_sql_file sql/corr_template.sql -v FIELD=unitsres +run_sql_file sql/corr_template.sql -v FIELD=unitstotal +run_sql_file sql/corr_inwoodrezoning.sql +run_sql_file sql/corr_template.sql -v FIELD=bct2020 +run_sql_file sql/corr_template.sql -v FIELD=address +run_sql_file sql/remove_unitlots.sql + +echo "Creating export tables" +run_sql_file sql/export.sql + +run_sql_file \ + sql/export_mappluto_shp.sql\ + -v TABLE='mappluto'\ + -v GEOM='clipped_2263' + +run_sql_file \ + sql/export_mappluto_shp.sql\ + -v TABLE='mappluto_unclipped'\ + -v GEOM='geom_2263' + +run_sql_file \ + sql/export_mappluto_gdb.sql\ + -v TABLE='mappluto_gdb'\ + -v GEOM='clipped_2263' + +run_sql_file \ + sql/export_mappluto_gdb.sql\ + -v TABLE='mappluto_unclipped_gdb'\ + -v GEOM='geom_2263' + echo 'Done' diff --git a/products/pluto/pluto_build/03_corrections.sh b/products/pluto/pluto_build/03_corrections.sh deleted file mode 100755 index 364c896649..0000000000 --- a/products/pluto/pluto_build/03_corrections.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -source ./bash/config.sh -set_error_traps - -run_sql_file sql/corr_create.sql - -echo "Applying corrections to PLUTO" -run_sql_file sql/corr_lotarea.sql -run_sql_file sql/corr_template.sql -v FIELD=yearbuilt -run_sql_file sql/corr_template.sql -v FIELD=ownername -run_sql_file sql/corr_ownername_punctuation.sql -run_sql_file sql/corr_template.sql -v FIELD=cd -run_sql_file sql/corr_template.sql -v FIELD=numfloors -run_sql_file sql/corr_template.sql -v FIELD=numbldgs -run_sql_file sql/corr_template.sql -v FIELD=unitsres -run_sql_file sql/corr_template.sql -v FIELD=unitstotal -run_sql_file sql/corr_inwoodrezoning.sql -run_sql_file sql/corr_template.sql -v FIELD=bct2020 -run_sql_file sql/corr_template.sql -v FIELD=address -run_sql_file sql/remove_unitlots.sql diff --git a/products/pluto/pluto_build/04_archive.sh b/products/pluto/pluto_build/04_archive.sh deleted file mode 100755 index d8d564a4d0..0000000000 --- a/products/pluto/pluto_build/04_archive.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -source ./bash/config.sh -set_error_traps - -echo 'Create Export' -run_sql_file sql/export.sql - -run_sql_file \ - sql/export_mappluto_shp.sql\ - -v TABLE='mappluto'\ - -v GEOM='clipped_2263' - -run_sql_file \ - sql/export_mappluto_shp.sql\ - -v TABLE='mappluto_unclipped'\ - -v GEOM='geom_2263' - -run_sql_file \ - sql/export_mappluto_gdb.sql\ - -v TABLE='mappluto_gdb'\ - -v GEOM='clipped_2263' - -run_sql_file \ - sql/export_mappluto_gdb.sql\ - -v TABLE='mappluto_unclipped_gdb'\ - -v GEOM='geom_2263' - -run_sql_command " - DROP TABLE IF EXISTS mappluto_sample; - SELECT * INTO mappluto_sample FROM mappluto_unclipped_gdb limit 5; - ALTER TABLE mappluto_sample ALTER COLUMN \"Borough\" SET NOT NULL; - ALTER TABLE mappluto_sample ALTER COLUMN \"Block\" SET NOT NULL; - ALTER TABLE mappluto_sample ALTER COLUMN \"Lot\" SET NOT NULL; - ALTER TABLE mappluto_sample ALTER COLUMN \"BBL\" SET NOT NULL; - ALTER TABLE mappluto_sample ALTER COLUMN \"BoroCode\" SET NOT NULL;" diff --git a/products/pluto/tests/assert_all_geometry_is_valid.sql b/products/pluto/tests/assert_all_geometry_is_valid.sql new file mode 100644 index 0000000000..3cdcd52a75 --- /dev/null +++ b/products/pluto/tests/assert_all_geometry_is_valid.sql @@ -0,0 +1,20 @@ +{{ + config( + tags = ['de_check', 'minor', 'major'], + meta = { + 'description': ''' + This test checks for any invalid lot geometries + ''', + 'next_steps': 'Contact DE and GIS to investigate' + } + ) +}} + +WITH export_pluto AS ( + SELECT * + FROM {{ source('build_sources', 'export_pluto') }} +) + +SELECT * +FROM export_pluto +WHERE zonedist NOT IN (SELECT zonedist FROM far_lookup)