912 better method for making sure jsonschema logging does not dump pii into logs

.github/workflows/autour-handler.yml

@@ -29,6 +29,9 @@ jobs:
       - name: Copy schemas
         working-directory: ./handlers/autour-handler
         run: cp -R ../../schemas src/
+      - name: Copy config
+        working-directory: ./handlers/autour-handler
+        run: cp -R ../../config src/
       - name: Install dependencies
         working-directory: ./handlers/autour-handler
         run: npm ci

.github/workflows/high-charts-handler.yml

@@ -29,6 +29,9 @@ jobs:
       - name: Copy schemas
         working-directory: ./handlers/high-charts
         run: cp -R ../../schemas src/
+      - name: Copy config
+        working-directory: ./handlers/high-charts
+        run: cp -R ../../config src/
       - name: Install dependencies
         working-directory: ./handlers/high-charts
         run: npm ci

.github/workflows/motd.yml

@@ -29,6 +29,9 @@ jobs:
       - name: Copy schemas
         working-directory: ./handlers/motd
         run: cp -R ../../schemas src/
+      - name: Copy config
+        working-directory: ./handlers/motd
+        run: cp -R ../../config src/
       - name: Install dependencies
         working-directory: ./handlers/motd
         run: npm ci

.github/workflows/nominatim-preprocessor.yml

@@ -29,6 +29,9 @@ jobs:
       - name: Copy schemas
         working-directory: ./preprocessors/nominatim
         run: cp -R ../../schemas src/
+      - name: Copy config
+        working-directory: ./preprocessors/nominatim
+        run: cp -R ../../config src/
       - name: Install dependencies
         working-directory: ./preprocessors/nominatim
         run: npm ci

.github/workflows/osm-streets-handler.yml

@@ -29,6 +29,9 @@ jobs:
       - name: Copy schemas
         working-directory: ./handlers/osm-streets-handler
         run: cp -R ../../schemas src/
+      - name: Copy config
+        working-directory: ./handlers/osm-streets-handler
+        run: cp -R ../../config src/
       - name: Install dependencies
         working-directory: ./handlers/osm-streets-handler
         run: npm ci

.github/workflows/photo-audio-handler.yml

@@ -29,6 +29,9 @@ jobs:
       - name: Copy schemas
         working-directory: ./handlers/photo-audio-handler
         run: cp -R ../../schemas src/
+      - name: Copy config
+        working-directory: ./handlers/photo-audio-handler
+        run: cp -R ../../config src/
       - name: Install dependencies
         working-directory: ./handlers/photo-audio-handler
         run: npm ci

.github/workflows/photo-audio-haptics-handler.yml

@@ -29,6 +29,9 @@ jobs:
       - name: Copy schemas
         working-directory: ./handlers/photo-audio-haptics-handler
         run: cp -R ../../schemas src/
+      - name: Copy config
+        working-directory: ./handlers/photo-audio-haptics-handler
+        run: cp -R ../../config src/
       - name: Install dependencies
         working-directory: ./handlers/photo-audio-haptics-handler
         run: npm ci

build.yml

@@ -96,11 +96,6 @@ services:
             context: .
             dockerfile: ./preprocessors/graphic-caption/Dockerfile
         image: "graphic-caption:latest"
-    text-followup:
-        build:
-            context: .
-            dockerfile: ./preprocessors/text-followup/Dockerfile
-        image: "text-followup:latest"
     emotion-recognition:
         build:
             context: .
@@ -268,11 +263,6 @@ services:
             context: .
             dockerfile: ./handlers/svg-action-recognition/Dockerfile
         image: "svg-action-recognition:latest"
-    text-followup-handler:
-        build:
-            context: .
-            dockerfile: ./handlers/text-followup-handler/Dockerfile
-        image: "text-followup-handler:latest"
     # Supercollider on Fedora 34
     supercollider-base:
         build:

config/__init__.py

@@ -0,0 +1,2 @@
+# /app/config/__init__.py
+# This file marks the config directory as a Python package.

config/logging_utils.py

@@ -0,0 +1,34 @@
+import logging
+import os
+
+# Define a PII log level
+PII_LOG_LEVEL = 5
+logging.addLevelName(PII_LOG_LEVEL, "PII")
+
+
+def pii(self, message, *args, **kwargs):
+    if self.isEnabledFor(PII_LOG_LEVEL):
+        self._log(PII_LOG_LEVEL, message, args, **kwargs)
+
+
+logging.Logger.pii = pii
+logging.pii = lambda msg, *args, **kwargs: logging.log(
+    PII_LOG_LEVEL, msg, *args, **kwargs)
+
+
+def configure_logging():
+    """
+    Configures logging based on environment variables.
+    """
+    log_level = os.getenv("LOG_LEVEL", "INFO").upper()
+    pii_logging_enabled = os.getenv("PII_LOGGING_ENABLED",
+                                    "false").lower() == "true"
+
+    level = getattr(logging, log_level, logging.INFO)
+    logging.basicConfig(level=level)
+
+    if pii_logging_enabled:
+        logging.warning("Environment Unicorn: PII logging enabled!")
+        logging.getLogger().setLevel(PII_LOG_LEVEL)  # Lower log level to PII
+    else:
+        logging.info("Environment Pegasus: PII logging is disabled.")

config/logging_utils.ts

@@ -0,0 +1,47 @@
+// https://medium.com/@jaiprajapati3/masking-of-sensitive-data-in-logs-700850e233f5
+// https://github.com/winstonjs/winston/blob/c69cdb0cec15a138e0b6e374501e027d1c39606c/index.d.ts
+
+import winston, { LogMethod, LogEntry } from 'winston';
+const logger = winston.createLogger({
+    level: 'info',
+    format: winston.format.json(),
+    transports: [
+        new winston.transports.Console({
+            format: winston.format.simple(),
+        }),
+    ],
+});
+
+interface PIILogger extends LogMethod {
+    pii: (message: string) => void;
+}
+
+const piiLoggingEnabled = process.env.PII_LOGGING_ENABLED === 'true';
+
+if (piiLoggingEnabled) {
+    logger.warn("Environment Unicorn: PII logging enabled!");
+} else {
+    logger.info("Environment Pegasus: PII logging is disabled.");
+}
+
+// PII Logger Function
+const piiLogger: PIILogger = ((arg1: string | LogEntry, arg2?: any) => {
+    if (typeof arg1 === 'string' && arg2 !== undefined) { // log when both level and message are provided
+        logger.log(arg1, arg2);
+    } else if (typeof arg1 === 'object') { // log when an object is provided
+        logger.log(arg1);
+    } else {
+        throw new Error('Invalid log format');
+    }
+}) as PIILogger;
+
+// Function for logging PII-related errors
+piiLogger.pii = (message: string) => {
+    if (piiLoggingEnabled) {
+        logger.error(`[PII] ${message}`);
+    } else {
+        logger.warn("PII logging attempted but is DISABLED.");
+    }
+};
+
+export { piiLogger };

docker-compose.yml

@@ -104,17 +104,6 @@ services:
       env_file:
         ./config/ollama.env
 
-  text-followup:
-    profiles: [test, default]
-    image: ghcr.io/shared-reality-lab/image-preprocessor-text-followup:${REGISTRY_TAG}
-    restart: unless-stopped
-    labels:
-      ca.mcgill.a11y.image.preprocessor: 1
-      ca.mcgill.a11y.image.port: 5000
-      ca.mcgill.a11y.image.cacheTimeout: 0
-      ca.mcgill.a11y.image.route: "followup"
-    env_file:
-      ./config/ollama.env
 
   graphic-tagger:
     profiles: [production, test, default]
@@ -302,14 +291,6 @@ services:
     labels:
         ca.mcgill.a11y.image.handler: enable
 
-  text-followup-handler:
-    profiles: [test, default]
-    image: ghcr.io/shared-reality-lab/image-handler-text-followup:${REGISTRY_TAG}
-    restart: unless-stopped
-    labels:
-        ca.mcgill.a11y.image.handler: enable
-        ca.mcgill.a11y.image.route: "followup"
-
 # end - common services
 
 # start - unicorn exclusive services

handlers/autour-handler/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:alpine as builder
+FROM node:alpine AS builder
 
 RUN addgroup -g 4322 storage && addgroup node storage
 
@@ -9,9 +9,10 @@ RUN npm ci
 COPY /handlers/autour-handler/src ./src
 COPY /handlers/autour-handler/types ./types
 COPY /schemas src/schemas
+COPY /config ./src/config
 RUN npm run build && npm prune --production
 
-FROM node:alpine as final
+FROM node:alpine AS final
 
 RUN apk add --no-cache curl