feat(cloudflare_logpush): add SourceInternalIP to related.ip field(#15088)

Update gateway_http and gateway_network data streams to include the
Cloudflare SourceInternalIP field value in the related.ip array for
better IP correlation and analysis. The internal source IP provides
additional context about the originating device within the network.

Co-authored-by: Rory B <[email protected]>
Co-authored-by: Dan Kortschak <[email protected]>
Co-authored-by: Andrew Kroh <[email protected]>

Author: 4 people

packages/cloudflare_logpush/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.41.0"
+  changes:
+    - description: Add Cloudflare `SourceInternalIP` field values to `related.ip` for the gateway_http and gateway_network data streams.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15088
 - version: "1.40.0"
   changes:
     - description: Added support for Azure Blob Storage input for all data streams.

packages/cloudflare_logpush/data_stream/gateway_http/_dev/test/pipeline/test-pipeline-gateway-http.log-expected.json

@@ -129,7 +129,8 @@
                 ],
                 "ip": [
                     "67.43.156.2",
-                    "89.160.20.129"
+                    "89.160.20.129",
+                    "192.168.1.123"
                 ],
                 "user": [
                     "166befbb-00e3-5e20-bd6e-27245723949f",
@@ -298,7 +299,8 @@
                 ],
                 "ip": [
                     "67.43.156.2",
-                    "89.160.20.129"
+                    "89.160.20.129",
+                    "192.168.1.123"
                 ],
                 "user": [
                     "166befbb-00e3-5e20-bd6e-27245723949f",
@@ -467,7 +469,8 @@
                 ],
                 "ip": [
                     "67.43.156.2",
-                    "89.160.20.129"
+                    "89.160.20.129",
+                    "192.168.1.123"
                 ],
                 "user": [
                     "166befbb-00e3-5e20-bd6e-27245723949f",

packages/cloudflare_logpush/data_stream/gateway_http/elasticsearch/ingest_pipeline/default.yml

@@ -392,6 +392,11 @@ processors:
       value: "{{{destination.ip}}}"
       if: ctx.destination?.ip != null
       allow_duplicates: false
+  - append:
+      field: related.ip
+      value: "{{{cloudflare_logpush.gateway_http.source.internal_ip}}}"
+      if: ctx.cloudflare_logpush?.gateway_http?.source?.internal_ip != null && ctx.cloudflare_logpush.gateway_http.source.internal_ip != ''
+      allow_duplicates: false
   - append:
       field: related.hosts
       value: "{{{host.id}}}"

packages/cloudflare_logpush/data_stream/gateway_network/_dev/test/pipeline/test-pipeline-gateway-network.log-expected.json

@@ -91,7 +91,8 @@
                 "ip": [
                     "67.43.156.2",
                     "89.160.20.129",
-                    "175.16.199.4"
+                    "175.16.199.4",
+                    "192.168.1.3"
                 ],
                 "user": [
                     "166befbb-00e3-5e20-bd6e-27245723949f",
@@ -219,7 +220,8 @@
                 "ip": [
                     "67.43.156.2",
                     "89.160.20.129",
-                    "175.16.199.4"
+                    "175.16.199.4",
+                    "192.168.1.3"
                 ],
                 "user": [
                     "166befbb-00e3-5e20-bd6e-27245723949f",
@@ -347,7 +349,8 @@
                 "ip": [
                     "67.43.156.2",
                     "89.160.20.129",
-                    "175.16.199.4"
+                    "175.16.199.4",
+                    "192.168.1.3"
                 ],
                 "user": [
                     "166befbb-00e3-5e20-bd6e-27245723949f",

packages/cloudflare_logpush/data_stream/gateway_network/elasticsearch/ingest_pipeline/default.yml

@@ -292,6 +292,11 @@ processors:
       value: "{{{cloudflare_logpush.gateway_network.override.ip}}}"
       if: ctx.cloudflare_logpush?.gateway_network?.override?.ip != null
       allow_duplicates: false
+  - append:
+      field: related.ip
+      value: "{{{cloudflare_logpush.gateway_network.source.internal_ip}}}"
+      if: ctx.cloudflare_logpush?.gateway_network?.source?.internal_ip != null && ctx.cloudflare_logpush.gateway_network.source.internal_ip != ''
+      allow_duplicates: false
   - append:
       field: related.hosts
       value: "{{{destination.domain}}}"

packages/cloudflare_logpush/data_stream/workers_trace/_dev/test/system/test-http-endpoint-config.yml

@@ -9,4 +9,4 @@ data_stream:
     listen_port: 9560
     preserve_original_event: true
     preserve_duplicate_custom_fields: true
-    enable_request_tracer: true
+    enable_request_tracer: true

packages/cloudflare_logpush/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: cloudflare_logpush
 title: Cloudflare Logpush
-version: "1.40.0"
+version: "1.41.0"
 description: Collect and parse logs from Cloudflare API with Elastic Agent.
 type: integration
 categories: