setting up pipeline

Author: prowe

.github/workflows/main.yml

@@ -0,0 +1,41 @@
+name: Build Pipeline
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+    
+jobs:
+  build-image:
+    name: Build the DBT Image
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6
+        with:
+          push: ${{ github.event_name != 'pull_request' }}
+
+  deploy-cloudformation:
+    name: Deploy CloudFormation Stack
+    runs-on: ubuntu-latest
+    needs: build-image
+    steps:
+      - uses: actions/checkout@v3
+      
+      - uses: aws-actions/setup-sam@v2
+        with:
+          use-installer: true
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-region: us-east-1
+          role-to-assume: 'arn:aws:iam::144406111952:role/Deployer'
+          role-session-name: Deploy
+
+      - name: Deploy stack
+        run: sam deploy

.github/workflows/validate_on_platforms.yml

@@ -1,8 +1,14 @@
 FROM ghcr.io/dbt-labs/dbt-core:1.9.latest
 
+ENV PGPASSWORD ''
+ENV PGUSER postgres
+ENV PGHOST localhost
+ENV DATA_S3_PATH ''
+
 WORKDIR /app
 
 COPY requirements.txt ./
 RUN pip3 install --upgrade pip
 RUN pip3 install --requirement requirements.txt
 
+COPY . ./

Dockerfile

@@ -19,16 +19,22 @@ clean-targets:
 
 require-dbt-version: [">=1.0.0", "<2.0.0"]
 
-seeds: 
-  +docs:
-    node_color: '#cd7f32'
-
 models:
   jaffle_shop:
     +materialized: table
     staging:
       +materialized: view
+      +database: memory
       +docs:
         node_color: 'silver'
     +docs:
       node_color: 'gold'
+
+on-run-start:
+#   - |
+#     CREATE OR REPLACE SECRET secret (
+#       TYPE s3,
+#       PROVIDER credential_chain
+#     );
+  - ATTACH 'ducklake:postgres:host={{ env_var('PGHOST') }}' AS ducklake (DATA_PATH '{{ env_var('DATA_S3_PATH' )}}');
+  - USE ducklake;

dbt_project.yml

@@ -3,7 +3,6 @@ version: 2
 models:
   - name: checkbook
     description: State of Iowa checkbook
-
     columns:
       - name: rec_no
       - name: fiscal_year

models/schema.yml

@@ -4,8 +4,14 @@ jaffle_shop:
   outputs:
     dev:
       type: duckdb
-      path: 'jaffle_shop.duckdb'
-      threads: 24
       extensions:
+        - aws
+        - https
         - httpfs
+        - ducklake
         - parquet
+        - postgres
+      secrets:
+        - type: s3
+          provider: credential_chain
+          region: "us-east-1"

profiles.yml

@@ -70,7 +70,7 @@ diff-cover==8.0.3
     # via sqlfluff
 duckcli==0.2.1
     # via -r requirements.in
-duckdb==1.2.0
+duckdb==1.3.0
     # via
     #   dbt-duckdb
     #   duckcli