feat: Preserve event.original when errors occur in pipelines

- Added append processor to global on_failure to preserve event original
- Added append processor to default piplines to preserve event original if error.message is set

Author: taylor-swanson

packages/arista_ngfw/changelog.yml

@@ -3,7 +3,7 @@
   changes:
     - description: Preserve event.original on pipeline error.
       type: enhancement
-      link: https://github.com/elastic/integrations/pull/1
+      link: https://github.com/elastic/integrations/pull/15798
 - version: "1.4.2"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/admin_login.yml

@@ -77,3 +77,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -987,6 +987,12 @@ processors:
         - _conf
         - _ingest
       ignore_missing: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - set:
       field: event.kind
@@ -998,3 +1004,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/http_event.yml

@@ -130,3 +130,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/interface_stats.yml

@@ -50,3 +50,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/intrusion_prevention.yml

@@ -69,3 +69,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/session_stats.yml

@@ -37,3 +37,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/system_stats.yml

@@ -148,3 +148,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/web_filter.yml

@@ -40,3 +40,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/cef/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.22.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15798
 - version: "2.21.3"
   changes:
     - description: Generate processor tags and normalize error handler.