Add Elk Stack to Cantabular Import

Author: PauloPortugal

.gitignore

@@ -13,3 +13,5 @@ cantabular-import/analysis/count-log-events/*.txt
 cantabular-import/analysis/tmp/*.txt
 
 cantabular-import/analysis/simple-plot/*.svg
+
+cantabular-import/elk/elasticsearch_data/*

cantabular-import/.env

@@ -1,3 +1,3 @@
-COMPOSE_FILE=deps.yml:dp-import-api.yml:dp-import-cantabular-dataset.yml:dp-import-cantabular-dimension-options.yml:dp-cantabular-server.yml:dp-cantabular-api-ext.yml:dp-dataset-api.yml:dp-cantabular-csv-exporter.yml:dp-cantabular-dimension-api.yml:dp-cantabular-filter-flex-api.yml:dp-cantabular-metadata-service.yml:dp-cantabular-metadata-exporter.yml:dp-cantabular-xlsx-exporter.yml:dp-recipe-api.yml:dp-filter-api.yml:zebedee.yml:dp-download-service.yml:dp-frontend-dataset-controller.yml:dp-api-router.yml:dp-frontend-router.yml:dp-publishing-dataset-controller.yml:florence.yml:the-train.yml:babbage.yml:dp-frontend-filter-flex-dataset.yml:dp-population-types-api.yml
+COMPOSE_FILE=elk.yml:deps.yml:dp-import-api.yml:dp-import-cantabular-dataset.yml:dp-import-cantabular-dimension-options.yml:dp-cantabular-server.yml:dp-cantabular-api-ext.yml:dp-dataset-api.yml:dp-cantabular-csv-exporter.yml:dp-cantabular-dimension-api.yml:dp-cantabular-filter-flex-api.yml:dp-cantabular-metadata-service.yml:dp-cantabular-metadata-exporter.yml:dp-cantabular-xlsx-exporter.yml:dp-recipe-api.yml:dp-filter-api.yml:zebedee.yml:dp-download-service.yml:dp-frontend-dataset-controller.yml:dp-api-router.yml:dp-frontend-router.yml:dp-publishing-dataset-controller.yml:florence.yml:the-train.yml:babbage.yml:dp-frontend-filter-flex-dataset.yml:dp-population-types-api.yml
 COMPOSE_PATH_SEPARATOR=:
 COMPOSE_PROJECT_NAME=cantabular-import-journey

cantabular-import/.env.backend

@@ -1,3 +1,3 @@
-COMPOSE_FILE=deps.yml:dp-import-api.yml:dp-import-cantabular-dataset.yml:dp-import-cantabular-dimension-options.yml:dp-cantabular-server.yml:dp-cantabular-api-ext.yml:dp-dataset-api.yml:dp-cantabular-csv-exporter.yml:dp-cantabular-dimension-api.yml:dp-filter-api.yml:dp-cantabular-filter-flex-api.yml:dp-cantabular-metadata-service.yml:dp-cantabular-metadata-exporter.yml:dp-cantabular-xlsx-exporter.yml:dp-recipe-api.yml:zebedee.yml:dp-download-service.yml:dp-population-types-api.yml
+COMPOSE_FILE=elk.yml:deps.yml:dp-import-api.yml:dp-import-cantabular-dataset.yml:dp-import-cantabular-dimension-options.yml:dp-cantabular-server.yml:dp-cantabular-api-ext.yml:dp-dataset-api.yml:dp-cantabular-csv-exporter.yml:dp-cantabular-dimension-api.yml:dp-filter-api.yml:dp-cantabular-filter-flex-api.yml:dp-cantabular-metadata-service.yml:dp-cantabular-metadata-exporter.yml:dp-cantabular-xlsx-exporter.yml:dp-recipe-api.yml:zebedee.yml:dp-download-service.yml:dp-population-types-api.yml
 COMPOSE_PATH_SEPARATOR=:
 COMPOSE_PROJECT_NAME=cantabular-import-journey

cantabular-import/elk.yml

@@ -0,0 +1,39 @@
+version: '3.3'
+services:
+  logstash:
+    image: logstash:7.16.2
+    container_name: logstash
+    restart: always
+    volumes:
+    - ./elk/logstash/logstash.conf:/etc/logstash/logstash.conf
+    command: logstash -f /etc/logstash/logstash.conf
+    depends_on:
+      - elasticsearch
+    ports:
+    - '5044:5044'
+    environment:
+      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+
+  elasticsearch:
+    image: elasticsearch:7.16.2
+    container_name: elasticsearch
+    restart: always
+    volumes:
+    - ./elk/elasticsearch_data:/usr/share/elasticsearch/data/
+    environment:
+      ES_JAVA_OPTS: "-Xmx256m -Xms256m"
+      discovery.type: single-node
+    ports:
+    - '9200:9200'
+    - '9300:9300'
+
+  kibana:
+    image: kibana:7.16.2
+    container_name: kibana
+    restart: always
+    ports:
+    - '5601:5601'
+    environment:
+      - ELASTICSEARCH_URL=http://elasticsearch:9200
+    depends_on:
+      - elasticsearch

cantabular-import/elk/logstash/logstash.conf

@@ -0,0 +1,12 @@
+input {
+  tcp {
+    port => 5044
+  }
+}
+
+output {
+    elasticsearch {
+        hosts => ["http://elasticsearch:9200"]
+         index => "containers-%{+YYYY.MM.dd}"
+    }
+}