enable observation QA (#743)

* enable exclusion of observations noted by QA as invalid for bcfishpass purposes and identification of observations that are known to be releases or associated with releases

* tag fish observations that are found (by QA) to be probable releases

* flag releases based on source/source_ref string matching

* include steehead in barrier/observation qa tables and exclude releases from counts upstream of natural barriers

* fix line endings, fix classification of two observations

* fix observation exclusion query

Author: smnorris

.github/workflows/ng-prod.yaml

@@ -91,7 +91,7 @@ jobs:
           jobs/connect
           cd model/01_access
           ./02_model_access_anthropogenic.sh
-          psql $DATABASE_URL -v ON_ERROR_STOP=1 -f sql/load_qa_observations_naturalbarriers_ch_cm_co_pk_sk.sql
+          psql $DATABASE_URL -v ON_ERROR_STOP=1 -f sql/load_qa_observations_naturalbarriers.sql
   model-habitat-linear:
     runs-on: ubuntu-latest
     environment: ng-prod
@@ -125,7 +125,7 @@ jobs:
           aws s3 cp wsg_crossing_summary_current.csv s3://bchamp/wsg_crossing_summary_current.csv --acl public-read
           aws s3 cp wsg_linear_summary_current.csv s3://bchamp/wsg_linear_summary_current.csv --acl public-read
           jobs/release_fptwg
-          psql $DATABASE_URL -c "select * from bcfishpass.qa_observations_ch_cm_co_pk_sk" --csv > qa_observations_ch_cm_co_pk_sk.csv
-          psql $DATABASE_URL -c "select * from bcfishpass.qa_naturalbarriers_ch_cm_co_pk_sk" --csv > qa_naturalbarriers_ch_cm_co_pk_sk.csv
-          aws s3 cp qa_observations_ch_cm_co_pk_sk.csv s3://bchamp/qa_observations_ch_cm_co_pk_sk.csv --acl public-read
-          aws s3 cp qa_naturalbarriers_ch_cm_co_pk_sk.csv s3://bchamp/qa_naturalbarriers_ch_cm_co_pk_sk.csv --acl public-read
+          psql $DATABASE_URL -c "select * from bcfishpass.qa_observations_ch_cm_co_pk_sk_st" --csv > qa_observations_ch_cm_co_pk_sk_st.csv
+          psql $DATABASE_URL -c "select * from bcfishpass.qa_naturalbarriers_ch_cm_co_pk_sk_st" --csv > qa_naturalbarriers_ch_cm_co_pk_sk_st.csv
+          aws s3 cp qa_observations_ch_cm_co_pk_sk.csv s3://bchamp/qa_observations_ch_cm_co_pk_sk_st.csv --acl public-read
+          aws s3 cp qa_naturalbarriers_ch_cm_co_pk_sk.csv s3://bchamp/qa_naturalbarriers_ch_cm_co_pk_sk_st.csv --acl public-read

data/observation_exclusions.csv

@@ -0,0 +1,54 @@
+  -- update natural barrier/observation QA tables to 
+  --  - include steelhead
+  --  - flag likely releases
+  --  - include counts per species
+
+BEGIN;
+
+  CREATE TABLE bcfishpass.qa_observations_ch_cm_co_pk_sk_st (
+    observation_key             text primary key       ,
+    species_code                character varying(6)   ,
+    observation_date            date                   ,
+    activity_code               character varying(100) ,
+    activity                    character varying(300) ,
+    life_stage_code             character varying(100) ,
+    life_stage                  character varying(300) ,
+    acat_report_url             character varying(254) ,
+    agency_name                 character varying(60)  ,
+    source                      character varying(1000),
+    source_ref                  character varying(4000),
+    release                     boolean                ,
+    watershed_group_code        character varying(4)   ,
+    gradient_15_dnstr           text                   ,
+    gradient_20_dnstr           text                   ,
+    gradient_25_dnstr           text                   ,
+    gradient_30_dnstr           text                   ,
+    falls_dnstr                 text                   ,
+    subsurfaceflow_dnstr        text                   ,
+    gradient_15_dnstr_count     integer                ,
+    gradient_20_dnstr_count     integer,
+    gradient_25_dnstr_count     integer,
+    gradient_30_dnstr_count     integer,
+    falls_dnstr_count           integer,
+    subsurfaceflow_dnstr_count  integer
+  );
+
+  -- natural barriers to salmon with salmon observations upstream (exluding known/likely releases)
+  CREATE TABLE bcfishpass.qa_naturalbarriers_ch_cm_co_pk_sk_st (
+    barrier_id           text primary key,
+    barrier_type         text,
+    watershed_group_code text,
+    observations_upstr   text,
+    n_observations_upstr integer,
+    n_ch_upstr integer,
+    n_cm_upstr integer,
+    n_co_upstr integer,
+    n_pk_upstr integer,
+    n_sk_upstr integer,
+    n_st_upstr integer
+  );
+
+  DROP TABLE bcfishpass.qa_observations_ch_cm_co_pk_sk;
+  DROP TABLE bcfishpass.qa_naturalbarriers_ch_cm_co_pk_sk;
+
+COMMIT;

db/v0.7.8/barrier_observation_qa.sql

@@ -3,7 +3,13 @@ set -euxo pipefail
 
 PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1"
 
-# document the cabd fix tables
+# enable exclusion of individual fish observation records based on QA
+$PSQL -f observation_exclusions.sql
+
+# update observation/natural barrier qa tables
+$PSQL -f barrier_observation_qa.sql
+
+# document cabd fix tables
 $PSQL -f cabd_fix_table_comments.sql
 
 $PSQL -c "update bcfishpass.db_version set tag = '${PWD##*/}'"

db/v0.7.8/migrate.sh

@@ -0,0 +1,18 @@
+-- ADD FISH OBSERVATION EXCLUSIONS
+BEGIN;
+
+  CREATE TABLE bcfishpass.observation_exclusions (
+    observation_key       text ,
+    exclude               boolean,
+    release               boolean,
+    reviewer_name         text,
+    review_date           date,
+    source_1              text,
+    source_2              text,
+    notes                 text,
+    primary key (observation_key)         
+  );
+
+  ALTER TABLE bcfishpass.observations ADD COLUMN release boolean;
+
+COMMIT;

db/v0.7.8/observation_exclusions.sql

@@ -223,12 +223,17 @@ ogr2ogr \
     -sql "select
        observation_key,
        species_code,
+       species_name,
        observation_date,
+       agency_name,
+       source,
+       source_ref,
        activity_code,
        activity,
        life_stage_code,
        life_stage,
        acat_report_url,
+       release,
        linear_feature_id,
        blue_line_key,
        downstream_route_measure,

jobs/release_fptwg

@@ -41,6 +41,7 @@ begin;
   -- filter on species code and watershed group
   insert into bcfishpass.observations (
    observation_key,
+   release,
    fish_observation_point_id,
    species_code,
    agency_id,
@@ -76,6 +77,7 @@ begin;
 )
   select
    o.observation_key,
+   x.release, -- note records at locations of ongoing/documented release programs
    o.fish_observation_point_id,
    r.species_code_remap as species_code,
    o.agency_id,
@@ -111,6 +113,24 @@ begin;
   from bcfishobs.observations o
   inner join wsg_spp on o.watershed_group_code = wsg_spp.watershed_group_code
   inner join species_code_remap r on o.species_code = r.species_code
-  and array[o.species_code]::text[] && wsg_spp.species_codes;
+  and array[o.species_code]::text[] && wsg_spp.species_codes
+  -- exclude observations identified by QA as invalid (for fish passage modelling purposes)
+  left outer join bcfishpass.observation_exclusions x on o.observation_key = x.observation_key
+  where coalesce(x.exclude, false) is false;
+
+  -- use source_ref to find and tag records related to stocking/releases
+  
+  -- note that this source seems to be obsolete - was present nov 2025 but is absent from table as of jan 05 2026
+  UPDATE bcfishpass.observations
+  SET release = true
+  WHERE substring(source from 1 for position(':' in source)) = 'Releases Database:';  
+
+  UPDATE bcfishpass.observations
+  SET release = true
+  WHERE source_ref ilike '%BC RELEASE RECORDS DATABASE%';
+
+  UPDATE bcfishpass.observations
+  SET release = true
+  WHERE source_ref ilike '%STOCKING%';
 
 commit;