[azure] update routing for signinlogs and springcloudlogs

packages/azure/_dev/build/docs/events.md

@@ -104,7 +104,7 @@ Use the following table to identify the target data streams for each log categor
 | `logs-azure.graphactivitylog-*`    | `MicrosoftGraphActivityLogs`                                                                                                                                 |
 | `logs-azure.identity_protection-*` | `RiskyUsers`, `UserRiskEvents`                                                                                                                               |
 | `logs-azure.provisioning-*`        | `ProvisioningLogs`                                                                                                                                           |
-| `logs-azure.signinlogs-*`          | `SignInLogs`, `NonInteractiveUserSignInLogs`, `ServicePrincipalSignInLogs`, `ManagedIdentitySignInLogs`                                                      |
+| `logs-azure.signinlogs-*`          | `*SignInLogs`                                                      |
 | `logs-azure.springcloudlogs-*`     | `ApplicationConsole`, `SystemLogs`, `IngressLogs`, `BuildLogs`, `ContainerEventLogs`                                                                         |
 | `logs-azure.platformlogs-*`        | All other log categories                                                                                                                                     |
 

packages/azure/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.29.1"
+  changes:
+    - description: Interim fix to support non-standard log events.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/15205
 - version: "1.29.0"
   changes:
     - description: Parse responseBody and requestBody json in activitylogs.

packages/azure/data_stream/events/_dev/test/pipeline/test-signinlogs-raw.log

@@ -1,4 +1,5 @@
 {"Category":"SignInLogs"}
 {"Category":"NonInteractiveUserSignInLogs"}
 {"Category":"ServicePrincipalSignInLogs"}
+{"Category":"MicrosoftServicePrincipalSignInLogs"}
 {"Category":"ManagedIdentitySignInLogs"}

packages/azure/data_stream/events/_dev/test/pipeline/test-signinlogs-raw.log-expected.json

@@ -45,6 +45,21 @@
             },
             "message": "{\"Category\":\"ServicePrincipalSignInLogs\"}"
         },
+        {
+            "data_stream": {
+                "dataset": "azure.signinlogs",
+                "namespace": "default",
+                "type": "logs"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "dataset": "azure.signinlogs",
+                "kind": "event"
+            },
+            "message": "{\"Category\":\"MicrosoftServicePrincipalSignInLogs\"}"
+        },
         {
             "data_stream": {
                 "dataset": "azure.signinlogs",

packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml

@@ -102,11 +102,12 @@ processors:
   - set:
       field: event.dataset
       value: azure.signinlogs
-      if: 'ctx.tmp_json?.category == "SignInLogs" || ctx.tmp_json?.category == "NonInteractiveUserSignInLogs" || ctx.tmp_json?.category == "ServicePrincipalSignInLogs" || ctx.tmp_json?.category == "ManagedIdentitySignInLogs"'
+      # Use same logic as the `signinlogs` stream that drops any document that doesn't end with `SignInLogs`.
+      if: 'ctx.tmp_json?.category.endsWith("SignInLogs")'
   - set:
       field: event.dataset
       value: azure.springcloudlogs
-      if: 'ctx.tmp_json?.category == "ApplicationConsole" || ctx.tmp_json?.category == "SystemLogs" || ctx.tmp_json?.category == "IngressLogs" || ctx.tmp_json?.category == "BuildLogs" || ctx.tmp_json?.category == "ContainerEventLogs"'
+      if: 'ctx.tmp_json?.category == "SystemLogs" || ctx.tmp_json?.category == "ApplicationConsole" || ctx.tmp_json?.category == "IngressLogs" || ctx.tmp_json?.category == "BuildLogs" || ctx.tmp_json?.category == "ContainerEventLogs"'
       description: 'Azure Spring Apps log categories (refs: https://learn.microsoft.com/en-us/azure/azure-monitor/reference/supported-logs/microsoft-appplatform-spring-logs)'
 
   # Remove the temporary field used to identify the event dataset.

packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml

@@ -17,7 +17,7 @@ processors:
       value: azure.springcloudlogs
       ignore_failure: true
   - drop:
-      if: "ctx.azure.springcloudlogs.category != 'SystemLogs' && ctx.azure.springcloudlogs.category != 'ApplicationConsole'"
+      if: 'ctx.azure.springcloudlogs.category != "SystemLogs" && ctx.azure.springcloudlogs.category != "ApplicationConsole" && ctx.azure.springcloudlogs.category != "IngressLogs" && ctx.azure.springcloudlogs.category != "BuildLogs" && ctx.azure.springcloudlogs.category != "ContainerEventLogs"'
       ignore_failure: true
   - rename:
       field: azure.springcloudlogs.LogFormat

packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log

@@ -1,2 +1,5 @@
 {"Level":"4","callerIpAddress":"81.2.69.144","category":"SignInLogs","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","durationMs":0,"identity":"Test LTest","location":"FR","operationName":"Sign-in activity","operationVersion":"1.0","properties":{"appDisplayName":"Office 365","appId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","clientAppUsed":"Browser","conditionalAccessStatus":"notApplied","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","createdDateTime":"2019-10-18T04:45:48.0729893-05:00","deviceDetail":{"browser":"Chrome 77.0.3865","deviceId":"","operatingSystem":"MacOs"},"id":"8a4de8b5-095c-47d0-a96f-a75130c61d53","ipAddress":"81.2.69.144","isInteractive":false,"location":{"city":"Champs-Sur-Marne","countryOrRegion":"FR","geoCoordinates":{"latitude":48.12341234,"longitude":2.12341234},"state":"Seine-Et-Marne"},"originalRequestId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","processingTimeInMilliseconds":239,"riskDetail":"none","riskLevelAggregated":"none","riskLevelDuringSignIn":"none","riskState":"none","servicePrincipalId":"","status":{"errorCode":50140,"failureReason":"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in."},"tokenIssuerName":"","tokenIssuerType":"AzureAD","userDisplayName":"Test LTest","userId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","userPrincipalName":"[email protected]"},"resourceId":"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam","resultDescription":"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.","resultSignature":"None","resultType":"50140","tenantId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","time":"2019-10-18T09:45:48.0729893Z"}
-{"Level":"4","callerIpAddress":"81.2.69.144","category":"SignInLogs","correlationId":"a8d4eb85-90c5-740d-9af6-7a15036cd135","durationMs":0,"identity":"Test LTest","location":"FR","operationName":"Sign-in activity","operationVersion":"1.0","properties":{"appDisplayName":"Office 365","appId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","clientAppUsed":"Browser","conditionalAccessStatus":"notApplied","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","createdDateTime":"2019-10-18T04:45:48.0729893-05:00","deviceDetail":{"browser":"Chrome 77.0.3865","deviceId":"","operatingSystem":"MacOs"},"id":"8a4de8b5-095c-47d0-a96f-a75130c61d53","ipAddress":"81.2.69.144","isInteractive":false,"location":{"city":"Champs-Sur-Marne","countryOrRegion":"FR","geoCoordinates":{"latitude":48.12341234,"longitude":2.12341234},"state":"Seine-Et-Marne"},"originalRequestId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","processingTimeInMilliseconds":239,"riskDetail":"none","riskLevelAggregated":"none","riskLevelDuringSignIn":"none","riskState":"none","servicePrincipalId":"","status":{"errorCode":50140,"failureReason":"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.","additionalDetails":"MFA required"},"tokenIssuerName":"","tokenIssuerType":"AzureAD","userDisplayName":"Test LTest","userId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","userPrincipalName":"c3813493-bf92-5123-2717-8a8b2979c38b"},"resourceId":"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam","resultDescription":"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.","resultSignature":"None","resultType":"50140","tenantId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","time":"2019-10-18T09:45:48.0729893Z"}
+{"Level":"4","callerIpAddress":"81.2.69.144","category":"NonInteractiveUserSignInLogs","correlationId":"a8d4eb85-90c5-740d-9af6-7a15036cd135","durationMs":0,"identity":"Test LTest","location":"FR","operationName":"Sign-in activity","operationVersion":"1.0","properties":{"appDisplayName":"Office 365","appId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","clientAppUsed":"Browser","conditionalAccessStatus":"notApplied","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","createdDateTime":"2019-10-18T04:45:48.0729893-05:00","deviceDetail":{"browser":"Chrome 77.0.3865","deviceId":"","operatingSystem":"MacOs"},"id":"8a4de8b5-095c-47d0-a96f-a75130c61d53","ipAddress":"81.2.69.144","isInteractive":false,"location":{"city":"Champs-Sur-Marne","countryOrRegion":"FR","geoCoordinates":{"latitude":48.12341234,"longitude":2.12341234},"state":"Seine-Et-Marne"},"originalRequestId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","processingTimeInMilliseconds":239,"riskDetail":"none","riskLevelAggregated":"none","riskLevelDuringSignIn":"none","riskState":"none","servicePrincipalId":"","status":{"errorCode":50140,"failureReason":"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.","additionalDetails":"MFA required"},"tokenIssuerName":"","tokenIssuerType":"AzureAD","userDisplayName":"Test LTest","userId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","userPrincipalName":"c3813493-bf92-5123-2717-8a8b2979c38b"},"resourceId":"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam","resultDescription":"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.","resultSignature":"None","resultType":"50140","tenantId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","time":"2019-10-18T09:45:48.0729893Z"}
+{"Level":"4","callerIpAddress":"81.2.69.144","category":"ServicePrincipalSignInLogs","correlationId":"a8d4eb85-90c5-740d-9af6-7a15036cd135","durationMs":0,"identity":"Test LTest","location":"FR","operationName":"Sign-in activity","operationVersion":"1.0","properties":{"appDisplayName":"Office 365","appId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","clientAppUsed":"Browser","conditionalAccessStatus":"notApplied","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","createdDateTime":"2019-10-18T04:45:48.0729893-05:00","deviceDetail":{"browser":"Chrome 77.0.3865","deviceId":"","operatingSystem":"MacOs"},"id":"8a4de8b5-095c-47d0-a96f-a75130c61d53","ipAddress":"81.2.69.144","isInteractive":false,"location":{"city":"Champs-Sur-Marne","countryOrRegion":"FR","geoCoordinates":{"latitude":48.12341234,"longitude":2.12341234},"state":"Seine-Et-Marne"},"originalRequestId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","processingTimeInMilliseconds":239,"riskDetail":"none","riskLevelAggregated":"none","riskLevelDuringSignIn":"none","riskState":"none","servicePrincipalId":"","status":{"errorCode":50140,"failureReason":"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.","additionalDetails":"MFA required"},"tokenIssuerName":"","tokenIssuerType":"AzureAD","userDisplayName":"Test LTest","userId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","userPrincipalName":"c3813493-bf92-5123-2717-8a8b2979c38b"},"resourceId":"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam","resultDescription":"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.","resultSignature":"None","resultType":"50140","tenantId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","time":"2019-10-18T09:45:48.0729893Z"}
+{"Level":"4","callerIpAddress":"81.2.69.144","category":"MicrosoftServicePrincipalSignInLogs","correlationId":"a8d4eb85-90c5-740d-9af6-7a15036cd135","durationMs":0,"identity":"Test LTest","location":"FR","operationName":"Sign-in activity","operationVersion":"1.0","properties":{"appDisplayName":"Office 365","appId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","clientAppUsed":"Browser","conditionalAccessStatus":"notApplied","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","createdDateTime":"2019-10-18T04:45:48.0729893-05:00","deviceDetail":{"browser":"Chrome 77.0.3865","deviceId":"","operatingSystem":"MacOs"},"id":"8a4de8b5-095c-47d0-a96f-a75130c61d53","ipAddress":"81.2.69.144","isInteractive":false,"location":{"city":"Champs-Sur-Marne","countryOrRegion":"FR","geoCoordinates":{"latitude":48.12341234,"longitude":2.12341234},"state":"Seine-Et-Marne"},"originalRequestId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","processingTimeInMilliseconds":239,"riskDetail":"none","riskLevelAggregated":"none","riskLevelDuringSignIn":"none","riskState":"none","servicePrincipalId":"","status":{"errorCode":50140,"failureReason":"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.","additionalDetails":"MFA required"},"tokenIssuerName":"","tokenIssuerType":"AzureAD","userDisplayName":"Test LTest","userId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","userPrincipalName":"c3813493-bf92-5123-2717-8a8b2979c38b"},"resourceId":"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam","resultDescription":"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.","resultSignature":"None","resultType":"50140","tenantId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","time":"2019-10-18T09:45:48.0729893Z"}
+{"Level":"4","callerIpAddress":"81.2.69.144","category":"ManagedIdentitySignInLogs","correlationId":"a8d4eb85-90c5-740d-9af6-7a15036cd135","durationMs":0,"identity":"Test LTest","location":"FR","operationName":"Sign-in activity","operationVersion":"1.0","properties":{"appDisplayName":"Office 365","appId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","clientAppUsed":"Browser","conditionalAccessStatus":"notApplied","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","createdDateTime":"2019-10-18T04:45:48.0729893-05:00","deviceDetail":{"browser":"Chrome 77.0.3865","deviceId":"","operatingSystem":"MacOs"},"id":"8a4de8b5-095c-47d0-a96f-a75130c61d53","ipAddress":"81.2.69.144","isInteractive":false,"location":{"city":"Champs-Sur-Marne","countryOrRegion":"FR","geoCoordinates":{"latitude":48.12341234,"longitude":2.12341234},"state":"Seine-Et-Marne"},"originalRequestId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","processingTimeInMilliseconds":239,"riskDetail":"none","riskLevelAggregated":"none","riskLevelDuringSignIn":"none","riskState":"none","servicePrincipalId":"","status":{"errorCode":50140,"failureReason":"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.","additionalDetails":"MFA required"},"tokenIssuerName":"","tokenIssuerType":"AzureAD","userDisplayName":"Test LTest","userId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","userPrincipalName":"c3813493-bf92-5123-2717-8a8b2979c38b"},"resourceId":"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam","resultDescription":"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.","resultSignature":"None","resultType":"50140","tenantId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","time":"2019-10-18T09:45:48.0729893Z"}