WIP Add Docker capabilities

.dockerfile/Dockerfile

@@ -0,0 +1,59 @@
+FROM python:3.7-alpine AS build
+RUN set -eu \
+    ;mkdir -p /tiffy \
+    ; python3 -m venv /tiffy/venv
+WORKDIR /tiffy
+COPY . .
+RUN set -eu \
+    ; rmdir /tiffy/.dockerfile \
+    ;/tiffy/venv/bin/pip3 install -r requirements.txt --no-cache-dir \
+    ;rm requirements.txt  \
+    ;ln -s /dev/stdout /tiffy/tiffy.py.log \
+    ;chown -R nobody:nobody /tiffy \
+    ;
+
+
+
+FROM python:3.7-alpine
+COPY --from=build /tiffy /tiffy
+ENV PATH=/tiffy/venv/bin:$PATH
+USER nobody
+WORKDIR /tiffy
+ENTRYPOINT [ "python", "tiffy.py" ]
+
+# Build Arguments
+    ARG BUILD_DATE
+    ARG GIT_REPO
+    ARG VCS_REF
+    ARG NAME="tiffy"
+    ARG DESCRIPTION="This docker container is an feed generator from DCSO TIE to MISP."
+    ARG AUTHOR="DCSO TI Team <[email protected]>"
+    ARG LICENSE="BSD-3-Clause"
+
+# Image Environment Variables
+    ENV NAME=${NAME} \
+        VERSION=${VCS_REF} \
+        BUILD_DATE=${BUILD_DATE}
+
+# Label
+    LABEL   org.opencontainers.image.created="${BUILD_DATE}" \
+            org.opencontainers.image.url="${GIT_REPO}" \
+            org.opencontainers.image.source="${GIT_REPO}" \
+            org.opencontainers.image.version="${VCS_REF}" \
+            org.opencontainers.image.revision="${VCS_REF}" \
+            org.opencontainers.image.vendor="${VENDOR}" \
+            org.opencontainers.image.title="${NAME}" \
+            org.opencontainers.image.description="${DESCRIPTION}" \
+            #org.opencontainers.image.documentation="${DOCUMENTATION}" \
+            org.opencontainers.image.authors="${AUTHOR}" \
+            org.opencontainers.image.licenses="${LICENSE}"
+
+# Default Environment Variables
+    ENV TIFFY_CONF_MISP_EVENTS_BASE_THREAT_LEVEL="3" \
+        TIFFY_CONF_MISP_EVENTS_BASE_CONFIDENCE="80" \
+        TIFFY_CONF_MISP_EVENTS_BASE_SEVERITY="2" \
+        TIFFY_CONF_MISP_EVENTS_PUBLISHED=false \
+        TIFFY_CONF_MISP_ATTRIBUTES_TO_IDS=false \
+        TIFFY_CONF_MISP_ATTRIBUTES_TAGGING=false \
+        TIFFY_PARAM_OUTPUT_FORMAT="MISP" \
+        TIFFY_PARAM_LOG_LEVEL="warning"        

.dockerignore

@@ -0,0 +1,6 @@
+.*
+*.md
+docker
+images
+Makefile
+LICENSE

.travis.yml

@@ -0,0 +1,40 @@
+language: minimal
+dist: xenial
+addons:
+  apt:
+    sources:
+      - docker-xenial
+
+env:
+  ADD_TAG=""
+
+before_install:
+# Login to hub.docker.com
+- echo "$DOCKER_PASS" | docker login -u $DOCKER_USER --password-stdin
+
+install:
+# Add docker-retag executable
+- wget -q https://github.com/joshdk/docker-retag/releases/download/0.0.2/docker-retag && chmod +x docker-retag
+
+script:
+# Build Image via kaniko
+- docker run
+    -v "$TRAVIS_BUILD_DIR":/workspace
+    -v $HOME/.docker:/kaniko/.docker
+  gcr.io/kaniko-project/executor:latest
+    --context=/workspace
+    --build-arg VCS_REF=$TRAVIS_COMMIT
+    --build-arg GIT_REPO=https://github.com/$TRAVIS_REPO_SLUG
+    --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d")
+    --verbosity=info
+    --destination=dcso/tiffy
+
+# Retag images for other tags
+- for i in $ADD_TAG;
+  do
+    ./docker-retag dcso/tiffy $i;
+  done
+
+# # don't notify me when things fail
+# notifications:
+#   email: false

Makefile

@@ -0,0 +1,11 @@
+
+IMAGE_NAME:=dcso/tiffy
+
+build:
+	docker build -t $(IMAGE_NAME) -f .dockerfile/Dockerfile .
+
+test: build
+	@echo
+	docker run --rm --name=tiffy $(IMAGE_NAME)
+	@echo
+	docker exec -ti tiffy /tiffy/venv/bin/pytest

docker/Readme.md

@@ -0,0 +1,62 @@
+## Docker
+
+### Usage
+
+Tiffy is automatically built on a daily base via Travis CI in an Docker container.
+Tiffy itself generate only the Feed from our TIE. It requires a webserver to provide it to MISP.
+
+#### docker run
+
+docker run \
+    --name tiffy \
+    -e ENV=VALUE \
+    dcso/tiffy:latest
+
+#### docker-compose
+
+Example file is in ./docker/docker-compose.yml
+
+### Customization
+
+#### Required Variables
+
+| Variable                          | Default | Example                                | Description                    |
+| --------------------------------- | ------- | -------------------------------------- | ------------------------------ |
+| TIFFY_CONF_TIE_APIURL             |         |  https://tie.dcso.de/v1/api            | URL to TIE.                    |
+| TIFFY_CONF_TIE_APIKEY             |         |  12345683127481209123789               | API token for TIE access       |
+| TIFFY_CONF_MISP_ORGANISATION_NAME |         |  ACME                                  | Name of your MISP organization |
+| TIFFY_CONF_MISP_ORGANISATION_UUID |         |  5804adw2-12fe-1234-34av-07lk82aw012a  | UUID of your MISP organization |
+
+#### Optional Variables
+
+| Variable                                 | Default              | Example                  | Description                                                       |
+| ---------------------------------------- | -------------------- | ------------------------ | ----------------------------------------------------------------- |
+| TIFFY_CONF_MISP_EVENTS_BASE_THREAT_LEVEL | 3                    |                          | IoC will get this threat level if it is added                     |
+| TIFFY_CONF_MISP_EVENTS_BASE_CONFIDENCE   | 80                   |                          | IoC will get this confidence if it is added                       |
+| TIFFY_CONF_MISP_EVENTS_BASE_SEVERITY     | 2                    |                          | IoC will get this severity if it is added                         |
+| TIFFY_CONF_MISP_EVENTS_PUBLISHED         | false                |                          | IoC will get published in MISP                                    |
+| TIFFY_CONF_MISP_ATTRIBUTES_TO_IDS        | false                |                          | Set IDS flag for this IoC                                         |
+| TIFFY_PARAM_TIE_SEEN_FIRST               |                      | YYYY-MM-DD               | Download only IoC which are first seen at ... and newer           |
+| TIFFY_PARAM_TIE_SEEN_LAST                |                      | YYYY-MM-DD               | Download only IoC which are last seen at ... and older            |
+| TIFFY_PARAM_TIE_ACTOR                    |                      | example1,example2        | Download only IoC with this actor                                 |
+| TIFFY_PARAM_TIE_CATEGORY                 |                      | example1,example2        | Download only IoC with this category                              |
+| TIFFY_PARAM_TIE_FAMILY                   |                      |   example1,example2      | Download only IoC with this family                                |
+| TIFFY_PARAM_TIE_SOURCE                   |                      | example1,example2        | Download only IoC from this source                                |
+| TIFFY_PARAM_TIE_SEVERITY_MIN             |                      | 2                        | Download only IoC with this minimum severity                      |
+| TIFFY_PARAM_TIE_SEVERITY_MAX             |                      | 4                        | Download only IoC with this maximum severity                      |
+| TIFFY_PARAM_TIE_CONFIDENCE_MIN           |                      | 2                        | Download only IoC with this minimum confidence                    |
+| TIFFY_PARAM_TIE_CONFIDENCE_MAX           |                      | 4                        | Download only IoC with this maximum confidence                    |
+| TIFFY_PARAM_TIE_MISP_EVENT_TAGS          | {\\"name\\":\\"tlp:amber\\"}| {\\"name\\":\\"tlp:amber\\"} | Tag Event with the defined tags                                   |
+| TIFFY_PARAM_OUTPUT_FORMAT                | MISP                 |                          | You can choose the output format of the feed.                     |
+| TIFFY_PARAM_TIE_DISABLE_DEFAULT_FILTER   | false                | true / false             | To disable the default TIE filter.                                |
+| TIFFY_PARAM_LOG_LEVEL                    | INFO                 |                          | Define one of these log levels: DEBUG, INFO, WARNING, ERROR, CRITICAL |
+| TIFFY_PARAM_LOG_DISABLE_CONSOLE          | false                | true / false             | Disables log output to stdout                                     |
+| TIFFY_PARAM_LOG_DISABLE_FILE             |   false              | true / false             | Disables log output to file                                       |
+| TIFFY_PARAM_LOG_FILE                     | "~/tiffy.log"        |                          | Define the log path                                               |
+
+#### Proxy Variables
+
+| Variable    | Default | Example                               | Description                              |
+| ----------- | ------- | ------------------------------------- | ---------------------------------------- |
+| HTTP_PROXY  |         |  http://10.8.0.1:8000                 | Set an Proxy server for HTTP connections |
+| HTTPS_PROXY |         |  https://<user>:<pass>@10.8.0.1:8000  | Set Proxy server for HTTPS connections   |

docker/docker-compose.yml

@@ -0,0 +1,43 @@
+version: '3'
+
+### Networks ###
+# networks: 
+#   misp-backend:
+#     driver: bridge
+#     driver_opts:
+#      com.docker.network.bridge.name: "mispbr0"
+#      com.docker.network.enable_ipv6: "false"
+#     ipam:
+#       config:
+#       - subnet: "192.168.47.0/28"
+
+### Volumes ###
+volumes:
+  tiffy_feed_data:
+
+### Services ###
+services:
+  ### TIFFY ###
+  tiffy:
+    image: dcso/tiffy
+    container_name: tiffy
+    restart: unless-stopped
+    volumes:
+    - tiffy_feed_data:/tiffy/feed
+    environment:
+    - TIFFY_PARAM_LOG_DISABLE_FILE=TRUE
+    - TIFFY_PARAM_LOG_LEVEL=INFO
+    # networks:
+    #   misp-backend:
+  web:
+    image: nginx:alpine
+    container_name: tiffy_web
+    restart: unless-stopped
+    ports:
+    - 8001:80
+    volumes:
+    - tiffy_feed_data:/usr/share/nginx/tiffy
+    - ./nginx.conf:/etc/nginx/conf.d/default.conf
+    # networks:
+    #   misp-backend:
+

docker/nginx.conf

@@ -0,0 +1,19 @@
+server {                                                                                                                 
+    listen       80;                                                                                                     
+
+    access_log  /dev/stdout  main;
+    error_log   /dev/stderr warn;                                                                 
+
+    location / {                                                                                                         
+        root   /usr/share/nginx/tiffy;
+        autoindex on;                                                                                     
+    }                                                                                                                    
+
+    # redirect server error pages to the static page /50x.html                                                           
+    #                                                                                                                    
+    error_page   500 502 503 504  /50x.html;                                                                             
+    location = /50x.html {                                                                                               
+        root   /usr/share/nginx/html;                                                                                    
+    }                                                                                                                    
+
+}

tiffy.py

@@ -127,7 +127,7 @@ def init(category, actor, family, source, first_seen, last_seen, event_tags, out
         loglvl = 'INFO'
     if log_file_path is None:
         log_file_path = sys.path[0]
-    TIELoader.init_logger(log_file_path, "tiffy.py", loglvl, disable_console_log, disable_file_log)
+    TIELoader.init_logger(log_file_path, "tiffy.py.log", loglvl, disable_console_log, disable_file_log)
     try:
 
         # Check date arguments