feat: Preserve event.original when errors occur in pipelines (#15803)

- Added append processor to global on_failure to preserve event original
- Added append processor to default pipelines to preserve event original if error.message is set

Affects the following integrations:

- citrix_waf
- endace
- fortinet_fortiedr
- fortinet_fortigate
- fortinet_fortimail
- fortinet_fortimanager
- fortinet_fortiproxy
- goflow2
- hashicorp_vault
- hpe_aruba_cx

Author: taylor-swanson

packages/citrix_waf/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.19.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15803
 - version: "1.18.3"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/citrix_waf/data_stream/log/elasticsearch/ingest_pipeline/cef.yml

@@ -135,3 +135,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/citrix_waf/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -166,6 +166,12 @@ processors:
         - _tmp
         - _conf
       ignore_missing: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - remove:
       field:
@@ -182,3 +188,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/citrix_waf/data_stream/log/elasticsearch/ingest_pipeline/native.yml

@@ -39,3 +39,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/citrix_waf/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: citrix_waf
 title: "Citrix Web App Firewall"
-version: "1.18.3"
+version: "1.19.0"
 description: Ingest events from Citrix Systems Web App Firewall.
 type: integration
 categories:

packages/endace/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.2.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15803
 - version: "0.1.2"
   changes:
     - description: Normalize error handler.

packages/endace/data_stream/flow/elasticsearch/ingest_pipeline/compatibility.yml

@@ -34,3 +34,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/endace/data_stream/flow/elasticsearch/ingest_pipeline/default.yml

@@ -109,6 +109,12 @@ processors:
     field: _conf
     ignore_missing: true
     tag: remove_conf
+- append:
+    tag: append_preserve_original_event_on_error
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false
+    if: ctx.error?.message != null
 
 on_failure:
   - append:
@@ -118,3 +124,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/endace/data_stream/flow/elasticsearch/ingest_pipeline/endace.yml

@@ -7,15 +7,15 @@ processors:
       value: "ip_conv={{ source.ip }}%26{{ destination.ip }}"
       if: (ctx.destination?.ip != null && ctx.destination?.ip != '') && (ctx.source?.ip != null && ctx.source?.ip != '')
       tag: endace conversation set
-  
-  - set: 
+
+  - set:
       description: "Set IP Conversation if only destination.ip is present"
       field: _conf.ip_conv
       value: "ip={{ destination.ip }}"
       if: (ctx.destination?.ip != null && ctx.destination.ip != '') && (ctx.source?.ip == null || ctx.source?.ip == '')
       tag: endace destination ip set
 
-  - set: 
+  - set:
       description: "Set IP Conversation if only source.ip is present"
       field: _conf.ip_conv
       value: "ip={{ source.ip }}"
@@ -90,3 +90,7 @@ on_failure:
       field: event.kind
       description: "Event Kind"
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/endace/data_stream/flow/elasticsearch/ingest_pipeline/geoip.yml

@@ -109,3 +109,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false