Merge pull request #198 from keboola/marek-CFT-3325-mTLS

Update: CFT-3325 - mTLS

Author: soustruh

.gitignore

@@ -14,3 +14,6 @@ vendor/
 /python-sync-actions/src/test.py
 /python-sync-actions/data
 __pycache__/
+docker/keys/*
+!docker/keys/genkeys.sh
+!docker/keys/keys-to-config-json.py

.python-version

@@ -0,0 +1 @@
+3.12

README.md

@@ -29,7 +29,7 @@ Example:
 
 ```
 https://yourDomain.zendesk.com/api/v2/
-```   
+```
 
 -- OR --
 
@@ -148,7 +148,7 @@ Moved to [new docs](https://developers.keboola.com/extend/generic-extractor/conf
 - **authentication.type**: `query`
 - **authentication.query.apiKey**: `{"attr": "apiKey"}`
     - this will look for the *apiKey* query parameter value in the config attribute named *apiKey*
-- **authentication.query.sig**: 
+- **authentication.query.sig**:
     ```
     {
         "function": "md5",
@@ -169,7 +169,7 @@ Moved to [new docs](https://developers.keboola.com/extend/generic-extractor/conf
             }
         ]
     }
-    ```   
+    ```
     - this will generate a *sig* parameter value from MD5 of merged configuration table attributes *apiKey* and *secret*, followed by current *time()* at the time of the request (time() being the PHP function)
     - Allowed functions are listed below in the *User functions* section
     - If you're using any config parameter by using `"attr": "parameterName"`, it has to be identical string to the one in the actual config, including eventual `#` if KBC Docker's encryption is used.
@@ -468,7 +468,7 @@ Moved to [new docs](https://developers.keboola.com/extend/generic-extractor/conf
     - sets which query parameter should contain the limit value (default to `limit`)
 - **pagination.offsetParam**(optional)
     - sets which query parameter should contain the offset value (default to `offset`)
-        
+
         ```
             "api": {
                 "pagination": {
@@ -478,7 +478,7 @@ Moved to [new docs](https://developers.keboola.com/extend/generic-extractor/conf
                     "offsetParam": "offset"
                 }
             }
-        ``` 
+        ```
 
 - **pagination.firstPageParams**(optional)
     - Whether or not include limit and offset params in the first request (default to `true`)
@@ -769,7 +769,7 @@ Moved to [new docs](https://developers.keboola.com/extend/generic-extractor/conf
                 }
             ]
         }
-        ```    
+        ```
     - **dataType**: Type of data returned by the endpoint. It also describes a table name, where the results will be stored
     - **dataField**: Allows to override which field of the response will be exported.
         - If there's multiple arrays in the response "root" the extractor may not know which array to export and fail
@@ -842,7 +842,7 @@ Moved to [new docs](https://developers.keboola.com/extend/generic-extractor/conf
                 "data": {"object": "can't really parse this!"}
             }
         ]
-        ```  
+        ```
 
         - To be able to work with such response, set `"responseFilter": "data"` - it should be a path within each object of the response array, **not** including the key of the response array
         - To filter values within nested arrays, use `"responseFilter": "data.array[].key"`
@@ -1218,14 +1218,77 @@ Best way to create and test new configurations is run extractor in docker contai
 # Running tests:
 ```
 docker compose run --rm tests
-``` 
+```
 
 or (with local source code and vendor copy)
 
 ```
 docker compose run --rm tests-local
-``` 
+```
+
+# Debugging mTLS support
+
+## Generating certificates
+
+Generate CA, server and client certificates:
+
+```
+# starting from project root
+cd docker/keys
+./genkeys.sh
+```
+
+The script also creates a `config.json` file with the following structure to be pasted into your own `config.json`:
+
+```
+"api": {
+    "baseUrl": "https://server.local/",
+    "caCertificate": "-- rootCA.crt --",
+    "#clientCertificate": "-- client.crt bundled with client.key --",
+}
+```
+
+## Testing in PHP
+
+```
+to be written…
+```
+
+## Testing in Python
+
+1. Run local nginx server:
+    ```
+    # starting from project root
+    cd python-sync-actions
+    docker compose up server.local
+    ```
+1. Create a `config.json` file in `python-sync-actions/data` with the following content:
+    ```
+    {
+        "parameters": {
+            "__SELECTED_JOB": "0",
+            "config": {
+                "jobs": [
+                    {
+                        "__NAME": "mTLS check",
+                        "endpoint": "",
+                        "method": "GET"
+                    }
+                ]
+            },
+            "api": {
+            }
+        }
+    }
+    ```
+1. Paste the `"api"` section generated in the `Generate certificates section` into the newly created config file.
+1. Run the python-sync-actions component:
+    ```
+    cd python-sync-actions
+    docker compose up dev
+    ```
+
 
-## License
+# License
 
 MIT licensed, see [LICENSE](./LICENSE) file.

docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3'
 services:
   app: &app
     build: .
@@ -35,3 +34,12 @@ services:
     links:
       - jsontest:jsontest-behind-proxy
 
+  server.local:
+    image: nginx:alpine
+    ports:
+      - "443:443"
+    volumes:
+      - ./docker/nginx/default.conf:/etc/nginx/conf.d/default.conf
+      - ./docker/keys/server.crt:/etc/nginx/server.crt
+      - ./docker/keys/server.key:/etc/nginx/server.key
+      - ./docker/keys/rootCA.crt:/etc/nginx/ca.crt

docker/keys/genkeys.sh

@@ -0,0 +1,17 @@
+echo "creating rootCA"
+openssl genrsa -out rootCA.key 4096
+openssl req -x509 -new -nodes -key rootCA.key -subj "/C=CZ/ST=CZ/O=authority" -days 1024 -out rootCA.crt
+
+echo "creating server keys"
+openssl genrsa -out server.key 2048
+# SAN is required as it is the main place where modern clients check host name (in fact CN can be ignored -- and is by e.g. chrome or requests)
+openssl req -new  -key server.key -subj "/C=CZ/ST=CZ/O=mytest/CN=server.local" -addext "subjectAltName=DNS:server.local" -out server.csr
+# Extensions such as SAN are not coppied by default from CSR when creating the certificate, -copy_extensions is required (semi-recent addition to OpenSSL)
+openssl x509 -req -in server.csr -CA rootCA.crt -CAkey rootCA.key -CAcreateserial -out server.crt -days 500 -copy_extensions copy
+
+echo "creating client keys"
+openssl genrsa -out client.key 2048
+openssl req -new -key client.key -subj "/C=CZ/ST=CZ/O=mytest/CN=client.local" -addext "subjectAltName=DNS:client.local" -out client.csr
+openssl x509 -req -in client.csr -CA rootCA.crt -CAkey rootCA.key -CAcreateserial -out client.crt -days 500 -copy_extensions copy
+
+python3 keys-to-config-json.py

docker/keys/keys-to-config-json.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+import json
+
+
+with open("rootCA.crt") as f:
+    ca_cert = f.read()
+
+with open("client.crt") as f:
+    client_cert = f.read()
+
+with open("client.key") as f:
+    client_key = f.read()
+
+with open("config.json", "w") as f:
+    json.dump(
+        {
+            "api": {
+                "baseUrl": "https://server.local/",
+                "caCertificate": ca_cert,
+                "#clientCertificate": client_cert + client_key,
+            }
+        },
+        f,
+        indent=4,
+    )

docker/nginx/default.conf

@@ -0,0 +1,17 @@
+server {
+    listen              443 ssl;
+    server_name         server.local;
+    ssl_certificate     /etc/nginx/server.crt;
+    ssl_certificate_key /etc/nginx/server.key;
+
+    ssl_client_certificate /etc/nginx/ca.crt;
+    ssl_verify_client      optional;
+
+    location / {
+        if ($ssl_client_verify != SUCCESS) {
+            return 403;
+        }
+
+        return 200 '{"name": "Nginx", "type": "server"}';
+    }
+}

python-sync-actions/Dockerfile

@@ -1,25 +1,19 @@
 FROM nikolaik/python-nodejs:python3.12-nodejs18
-ENV PYTHONIOENCODING utf-8
 
-COPY /src /code/src/
-COPY /tests /code/tests/
-COPY requirements.txt /code/requirements.txt
-COPY flake8.cfg /code/flake8.cfg
-
-# install gcc to be able to build packages - e.g. required by regex, dateparser, also required for pandas
-RUN apt-get update && apt-get install -y build-essential curl
+RUN apt-get update && apt-get install -y curl
 
-# Install curlconverter using npm
 RUN npm install --global curlconverter
 
-
 RUN pip install flake8
 
+COPY requirements.txt /code/requirements.txt
 RUN pip install -r /code/requirements.txt
 
+COPY flake8.cfg /code/flake8.cfg
 
+COPY /src /code/src/
+COPY /tests /code/tests/
 
 WORKDIR /code/
 
-
 CMD ["python", "-u", "/code/src/component.py"]

python-sync-actions/docker-compose.yml

@@ -1,11 +1,11 @@
-version: "2"
 services:
   # for development purposes
   dev:
     build: .
     volumes:
         - ./:/code
         - ./data:/data
+        - ../docker/keys:/code/keys
     environment:
       - KBC_DATADIR=./data
   test:
@@ -35,8 +35,20 @@ services:
     tty: true
     stdin_open: true
     ports:
-      - "8888:80"
+      - 8888:80
     volumes:
       - ./tests/calls:/examples/
     environment:
-      - KBC_EXAMPLES_DIR=/examples/
+      - KBC_EXAMPLES_DIR=/examples/
+
+  # i was about to create a common network with the generic-extractor compose file
+  # to avoid duplication, but that would create an unnecessary dependency
+  server.local:
+    image: nginx:alpine
+    ports:
+      - "443:443"
+    volumes:
+      - ../docker/nginx/default.conf:/etc/nginx/conf.d/default.conf
+      - ../docker/keys/server.crt:/etc/nginx/server.crt
+      - ../docker/keys/server.key:/etc/nginx/server.key
+      - ../docker/keys/rootCA.crt:/etc/nginx/ca.crt

python-sync-actions/requirements.in

@@ -0,0 +1,9 @@
+keboola.component
+dataconf
+keboola.http-client
+keboola.utils
+keboola.json-to-csv==0.0.12
+mock==5.1.0
+freezegun==1.5.1
+nested-lookup==0.2.25
+python-dateutil==2.9.0.post0