Merge pull request #64 from sourceallies/ci-publish

Ci publish

Author: GeneTinderholm

.github/workflows/pr-checks.yml

@@ -24,3 +24,12 @@ jobs:
 
       - name: Run unit tests
         run: make run-unit-tests
+
+      - name: setup poetry
+        run: |
+          rm ./src/glue_biscuit/**/test_*.py
+          python3 -m pip install --upgrade 'poetry==1.3.1'
+          mkdir -p ./dist
+          sed -i 's/GH_TAG_NAME/1.2.3/' ./pyproject.toml
+      - name: build
+        run: poetry build

.github/workflows/pypi-publish.yml

@@ -0,0 +1,24 @@
+name: Publish to pypi
+on: release
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: setup
+        run: |
+          rm ./src/framework/**/test_*.py
+          python3 -m pip install --upgrade 'poetry==1.3.1'
+          mkdir -p ./dist
+          sed -i 's/GH_TAG_NAME/${{ github.event.release.tag_name }}/' ./pyproject.toml
+
+      - name: build
+        run: poetry build
+
+      - name: Publish distribution 📦 to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

.gitignore

@@ -5,3 +5,4 @@
 **/.pytest_cache
 login
 PyGlue.zip
+dist/

pyproject.toml

@@ -0,0 +1,30 @@
+[tool.poetry]
+name = "glue-biscuit"
+version = "GH_TAG_NAME"
+description = ""
+authors = ["Gene Tinderholm <[email protected]>", "Paul Rowe <[email protected]"]
+readme = "README"
+packages = [{include = "glue_biscuit", from = "./src"}]
+
+[tool.poetry.dependencies]
+python = "^3.7"
+black = "22.3.0"
+boto3 = "1.21.29"
+cfn-flip = "1.3.0"
+botocore = "1.24.46"
+click = "8.1.3"
+jmespath = "1.0.1"
+mypy-extensions = "0.4.3"
+pathspec = "0.10.1"
+platformdirs = "2.5.2"
+python-dateutil = "2.8.2"
+s3transfer = "0.5.2"
+six = "1.16.0"
+tomli = "2.0.1"
+typing-extensions = "4.3.0"
+urllib3 = "1.26.12"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+

src/books_data_product/load_books.py

@@ -1,7 +1,7 @@
 from awsglue import DynamicFrame
 from awsglue.context import GlueContext
 from pyspark.sql import SparkSession, DataFrame
-from framework import get_job_arguments
+from glue_biscuit import get_job_arguments
 from pyspark.sql.functions import to_date, col
 
 

src/books_data_product/test_load_books.py

@@ -4,7 +4,7 @@
 from pyspark.sql import DataFrame
 from awsglue.context import GlueContext
 from unittest.mock import patch, Mock, call, ANY
-from framework.test import (
+from glue_biscuit.test import (
     DataFrameMatcher,
     DynamicFrameMatcher,
     spark_context,

src/customer_events/process_customer_events.py

@@ -3,7 +3,7 @@
 from pyspark.sql import SparkSession, DataFrame
 from pyspark.sql.functions import coalesce, when, isnull, lit
 from datetime import datetime
-from framework import merge_and_retain_last
+from glue_biscuit import merge_and_retain_last
 
 
 def load_events(glue_context: GlueContext) -> DataFrame:

src/customer_events/test_process_customer_events.py

@@ -8,23 +8,16 @@
     TimestampType,
     IntegerType,
     LongType,
-    MapType,
 )
-from awsglue.context import GlueContext
-from unittest.mock import patch, Mock, call, ANY
+from unittest.mock import patch, Mock
 import pytest
-from framework.test import (
+from glue_biscuit.test import (
     DataFrameMatcher,
-    DynamicFrameMatcher,
-    spark_context,
-    mock_glue_context,
 )
 from customer_events.process_customer_events import (
     main,
-    load_events,
-    load_customers,
-    save_customers,
 )
+from glue_biscuit.test import mock_glue_context, spark_context
 
 
 @pytest.fixture