Added support for structured logs in ECS format and logs collection to ELK

Author: Mikalai Alimenkou

compose.yaml

@@ -18,4 +18,45 @@ services:
       - "4318:4318"   # OTLP HTTP ingest
       - "5778:5778"   # Remote sampling (HTTP)
       - "9411:9411"   # Zipkin compatible ingest (optional)
-    restart: unless-stopped
+
+  elastic:
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.1.5
+    container_name: library-elastic
+    environment:
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+      - ES_JAVA_OPTS=-Xms1g -Xmx1g
+      - cluster.name=library-logs
+    ports:
+      - "9200:9200"
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://localhost:9200/_cluster/health || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 30
+      start_period: 40s
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:9.1.5
+    container_name: library-kibana
+    environment:
+      - ELASTICSEARCH_HOSTS=http://elastic:9200
+    ports:
+      - "5601:5601"
+    depends_on:
+      elastic:
+        condition: service_healthy
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.137.0
+    container_name: library-otel-collector
+    command: [ "--config=/etc/otelcol/config.yaml" ]
+    volumes:
+      - ./otel/otel-collector-config.yaml:/etc/otelcol/config.yaml:ro
+      - ./logs:/var/log/app
+    ports:
+      - "4315:4317"  # OTLP gRPC
+      - "4316:4318"  # OTLP HTTP
+    depends_on:
+      elastic:
+        condition: service_healthy

otel/otel-collector-config.yaml

@@ -0,0 +1,39 @@
+extensions:
+  file_storage:
+    directory: /var/log/app/.otel-file-storage
+    create_directory: true
+
+receivers:
+  filelog:
+    include: [ /var/log/app/library-logs.json ]
+    start_at: end
+    storage: file_storage
+    operators:
+      - type: json_parser
+        parse_from: body
+  otlp:
+    protocols:
+      http:
+      grpc:
+
+processors:
+  batch: {}
+
+exporters:
+  debug:
+    verbosity: detailed
+
+  elasticsearch:
+    endpoint: http://elastic:9200
+    mapping:
+      mode: raw
+    sending_queue:
+      enabled: true
+
+service:
+  extensions: [file_storage]
+  pipelines:
+    logs:
+      receivers: [filelog,otlp]
+      processors: [batch]
+      exporters: [elasticsearch, debug]

src/main/resources/application-admin.yaml

@@ -11,17 +11,14 @@ spring:
     enabled: true
 
 management:
+  observations:
+    annotations:
+      enabled: true
   otlp:
-    logging:
-      export:
-        enabled: false
     tracing:
       endpoint: # define Jaeger endpoint
       export:
         enabled: true
-  logging:
-    export:
-      enabled: false
   tracing:
     enabled: true
     sampling:
@@ -63,4 +60,8 @@ management:
     os:
       enabled: true
     process:
-      enabled: true
+      enabled: true
+  metrics:
+    distribution:
+      percentiles-histogram:
+        http.server.requests: true

src/main/resources/application-dev.yaml

@@ -27,4 +27,9 @@ logging:
   level:
     web: DEBUG
   include-application-group: false
-  include-application-name: false
+  include-application-name: false
+  structured:
+    ecs:
+      service:
+        environment: local-dev
+        node-name: laptop

src/main/resources/application.yaml

@@ -13,8 +13,8 @@ server:
       max-queue-capacity: 1000
     basedir: tomcat
     accesslog:
-        enabled: true
-        pattern: '%t %a "%r" %s (%D ms)'
+      enabled: true
+      pattern: '%t %a "%r" %s (%D ms)'
   compression:
     enabled: true
     min-response-size: 512
@@ -79,6 +79,9 @@ library:
 logging:
   file:
     name: library.log
+  structured:
+    format:
+      console: ecs
 
 springdoc:
   packages-to-scan: com.xpinjection.library.adaptors.api

src/main/resources/logback-spring.xml

@@ -1,7 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <configuration>
     <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
-    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
 
     <springProfile name="json-logs">
         <springProperty name="app" source="spring.application.name" defaultValue="app"/>
@@ -51,7 +50,17 @@
         </root>
     </springProfile>
 
-    <springProfile name="!json-logs">
+    <springProfile name="structured-logs">
+        <include resource="org/springframework/boot/logging/logback/structured-console-appender.xml"/>
+
+        <root level="INFO">
+            <appender-ref ref="CONSOLE"/>
+        </root>
+    </springProfile>
+
+    <springProfile name="!json-logs &amp; !structured-logs">
+        <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
+
         <root level="INFO">
             <appender-ref ref="CONSOLE"/>
         </root>