Cloud SIEM custom detection rules Cdocsification

config/_default/menus/main.en.yaml

@@ -7060,9 +7060,9 @@ menu:
       identifier: cloud_siem_custom_detection_rules
       weight: 202
     - name: Create Rule
-      url: security/cloud_siem/detect_and_monitor/custom_detection_rules/create_rule/real_time_rule
+      url: security/cloud_siem/detect_and_monitor/custom_detection_rules/create_rule
       parent: cloud_siem_custom_detection_rules
-      identifier: cloud_siem_real_time_rule
+      identifier: cloud_siem_create_custom_detection_rule
       weight: 2021
     - name: Anomaly
       url: security/cloud_siem/detect_and_monitor/custom_detection_rules/anomaly

content/.gitignore

@@ -31,3 +31,4 @@
 /en/synthetics/notifications/template_variables/mobile.md
 /en/synthetics/notifications/template_variables/multistep.md
 /en/synthetics/notifications/template_variables/api.md
+/en/security/cloud_siem/detect_and_monitor/custom_detection_rules/create_rule/_index.md

customization_config/en/option_groups/cloud_siem_custom_detection_rules.yaml

@@ -0,0 +1,96 @@
+cloud_siem_detection_rule_type_options:
+  - id: real_time_rule
+    default: true
+  - id: scheduled_rule
+  - id: historical_job
+
+# cloud_siem_detection_threshold_rule_type_options:
+#   - id: real_time_rule
+#     default: true
+#   - id: scheduled_rule
+#   - id: historical_job
+
+# cloud_siem_detection_new_value_rule_type_options:
+#   - id: real_time_rule
+#     default: true
+#   - id: scheduled_rule
+#   - id: historical_job
+
+# cloud_siem_detection_anomaly_rule_type_options:
+#   - id: real_time_rule
+#     default: true
+#   - id: scheduled_rule
+#   - id: historical_job
+
+# cloud_siem_detection_content_anomaly_rule_type_options:
+#   - id: real_time_rule
+#     default: true
+#   - id: scheduled_rule
+#   - id: historical_job
+
+# cloud_siem_detection_impossible_travel_rule_type_options:
+#   - id: real_time_rule
+#     default: true
+#   - id: scheduled_rule
+#   - id: historical_job
+
+# cloud_siem_detection_third_party_rule_type_options:
+#   - id: real_time_rule
+#     default: true
+#   - id: scheduled_rule
+#   - id: historical_job
+
+# cloud_siem_detection_sequence_rule_type_options:
+#   - id: real_time_rule
+#     default: true
+
+# cloud_siem_detection_signal_correlation_rule_type_options:
+#   - id: real_time_rule
+#     default: true
+#   - id: scheduled_rule
+
+cloud_siem_detection_real_time_rule_search_query_options:
+  - id: threshold
+    default: true
+  - id: new_value
+  - id: anomaly
+  - id: content_anomaly
+  - id: impossible_travel
+  - id: third_party
+  - id: sequence
+  - id: signal_correlation
+
+cloud_siem_detection_scheduled_rule_search_query_options:
+  - id: threshold
+    default: true
+  - id: new_value
+  - id: anomaly
+  - id: content_anomaly
+  - id: impossible_travel
+  - id: third_party
+  - id: signal_correlation
+
+cloud_siem_detection_historical_job_search_query_options:
+  - id: threshold
+    default: true
+  - id: new_value
+  - id: anomaly
+  - id: content_anomaly
+  - id: impossible_travel
+  - id: third_party
+
+cloud_siem_detection_rule_query_language_options:
+  - id: event_query
+    default: true
+  - id: sql
+
+# cloud_siem_detection_rule_search_query_options:
+#   - id: threshold
+#     default: true
+#   - id: new_value
+#   - id: anomaly
+#   - id: content_anomaly
+#   - id: impossible_travel
+#   - id: third_party
+#   - id: signal_correlation
+#   - id: sequence

customization_config/en/options/general.yaml

@@ -506,4 +506,40 @@ options:
   id: cpp
 
 - label: Elixir
-  id: elixir
+  id: elixir
+
+- id: real_time_rule
+  label: Real-time rule
+
+- id: scheduled_rule
+  label: Scheduled rule
+
+- id: historical_job
+  label: Historical job
+
+- id: new_value
+  label: New value
+
+- id: anomaly
+  label: Anomaly
+
+- id: content_anomaly
+  label: Content anomaly
+
+- id: impossible_travel
+  label: Impossible travel
+
+- id: third_party
+  label: Third party
+
+- id: signal_correlation
+  label: Signal correlation
+
+- id: sequence
+  label: Sequence
+
+- id: event_query
+  label: Event Query
+
+- id: sql
+  label: SQL

customization_config/en/traits/general.yaml

@@ -51,4 +51,19 @@ traits:
 - id: protocol
   label: Protocol
   type: text
-  internal_notes: For example, HTTP, gRPC, or SOCKS.
+  internal_notes: For example, HTTP, gRPC, or SOCKS.
+
+- id: cloud_siem_detection_rule_type
+  label: "Rule type"
+  type: text
+  internal_notes: The rule type a user wants to use when creating a custom detection rule for Cloud SIEM.
+
+- id: cloud_siem_detection_rule_search_query
+  label: "Search query"
+  type: text
+  internal_notes: The search query a user wants to use when creating a custom detection rule for Cloud SIEM.
+
+- id: cloud_siem_detection_rule_query_language
+  label: "Query language"
+  type: text
+  internal_notes: The search query language a user wants to use when creating a custom detection rule for Cloud SIEM.

layouts/shortcodes/mdoc/en/security/cloud_siem/add_calculated_fields.mdoc.md

@@ -0,0 +1,7 @@
+1. Click **Add** and select **Calculated fields**.
+1. In **Name your field**, enter a descriptive name that indicates the purpose of the calculated field.
+    - For example, if you want to combine users' first and last name into one field, you might name the calculated field `fullName`.
+1. In the **Define your formula** field, enter a formula or expression, which determines the result to be computed and stored as the value of the calculated field for each log event.
+    - See [Calculated Fields Expressions Language][701] for information on syntax and language constructs.
+
+[701]: /logs/explorer/calculated_fields/expression_language/

layouts/shortcodes/mdoc/en/security/cloud_siem/add_reference_tables.mdoc.md

@@ -0,0 +1,5 @@
+1. Click the **Add** button next to the query editor and select **Join with Reference Table**.
+1. In the **Inner join with reference table** dropdown menu, select your reference table.
+1. In the **where field** dropdown menu, select the log field to join on.
+1. Select the **IN** or **NOT IN** operator to filter in or filter out matching logs.
+1. In the **column** dropdown menu, select the column of the reference table to join on.

layouts/shortcodes/mdoc/en/security/cloud_siem/anomaly_query.mdoc.md

@@ -0,0 +1,6 @@
+1. (Optional) In the **Count** dropdown menu, select attributes whose unique values you want to count during the specified time frame.
+1. (Optional) In the **group by** dropdown menu, select attributes you want to group by.
+    - The defined `group by` generates a signal for each `group by` value.
+    - Typically, the `group by` is an entity (like user or IP). The `group by` can also join the queries together.
+    - Joining logs that span a time frame can increase the confidence or severity of the security signal. For example, if you want to detect a successful brute force attack, both successful and unsuccessful authentication logs must be correlated for a user.
+    - Anomaly detection inspects how the `group by` attribute has behaved in the past. If a `group by` attribute is seen for the first time (for example, the first time an IP is communicating with your system) and is anomalous, it does not generate a security signal because the anomaly detection algorithm has no historical data to compare with.

layouts/shortcodes/mdoc/en/security/cloud_siem/content_anomaly_options.mdoc.md

@@ -0,0 +1,9 @@
+In the **Content anomaly detection options** section, specify the parameters to assess whether a log is anomalous or not.
+- Content anomaly detection balances precision and sensitivity using several rule parameters that you can set:
+    1. Similarity threshold: Defines how dissimilar a field value must be to be considered anomalous (default: `70%`).
+    1. Minimum similar items: Sets how many similar historical logs must exist for a value to be considered normal (default: `1`).
+    1. Evaluation window: The time frame during which anomalies are counted toward a signal (for example, a 10-minute time frame).
+- These parameters help to identify field content that is both unusual and rare, filtering out minor or common variations.
+- See [Anomaly detection parameters][601] for more information.
+
+[601]: /security/cloud_siem/detect_and_monitor/custom_detection_rules/content_anomaly/#anomaly-detection-parameters

layouts/shortcodes/mdoc/en/security/cloud_siem/content_anomaly_query.mdoc.md

@@ -0,0 +1,9 @@
+1. In the **Detect anomaly** field, specify the fields whose values you want to analyze.
+1. In the **group by** field, specify the fields you want to group by.
+    - The defined `group by` generates a signal for each `group by` value.
+    - Typically, the `group by` is an entity (like user or IP). The `group by` can also join the queries together.
+    - Joining logs that span a time frame can increase the confidence or severity of the security signal. For example, to detect a successful brute force attack, both successful and unsuccessful authentication logs must be correlated for a user.
+1. In the **Learn for** dropdown menu, select the number of days for the learning period. During the learning period, the rule sets a baseline of normal field values and does not generate any signals.
+    {% alert level="info" %}
+    If the detection rule is modified, the learning period restarts at day `0`.
+    {% /alert %}

layouts/shortcodes/mdoc/en/security/cloud_siem/create_suppression.mdoc.md

@@ -0,0 +1,16 @@
+(Optional) Create a suppression or add the rule to an existing suppression to prevent a signal from getting generated in specific cases. For example, if a user `john.doe` is triggering a signal, but their actions are benign and you do not want signals triggered from this user, add the following query into the **Add a suppression query** field: `@user.username:john.doe`.
+
+### Create new suppression
+
+1. Enter a name for the suppression rule.
+1. (Optional) Enter a description.
+1. Enter a suppression query.
+1. (Optional) Add a log exclusion query to exclude logs from being analyzed. These queries are based on **log attributes**.
+   {% alert level="info" %}
+   The legacy suppression was based on log exclusion queries, but it is now included in the suppression rule's **Add a suppression query** step.
+   {% /alert %}
+
+### Add to existing suppression
+
+1. Click **Add to Existing Suppression**.
+1. Select an existing suppression in the dropdown menu.

layouts/shortcodes/mdoc/en/security/cloud_siem/enable_decrease_severity.mdoc.md

@@ -0,0 +1,3 @@
+Toggle **Decrease severity for non-production environments** if you want to prioritize production environment signals over non-production signals.
+- The severity of signals in non-production environments are decreased by one level from what is defined by the rule case.
+- The severity decrement is applied to signals with an environment tag starting with `staging`, `test`, or `dev`.

layouts/shortcodes/mdoc/en/security/cloud_siem/enable_group_by.mdoc.md

@@ -0,0 +1 @@
+Toggle the **Enable Optional Group By** section if you want to group events even when values are missing. If there is a missing value, a sample value is generated so that the log does not get excluded.

layouts/shortcodes/mdoc/en/security/cloud_siem/enable_instantaneous_baseline.mdoc.md

@@ -0,0 +1 @@
+Toggle **Enable instantaneous baseline** if you want to build the baseline based on past events for the first event received.

layouts/shortcodes/mdoc/en/security/cloud_siem/forget_value.mdoc.md

@@ -0,0 +1 @@
+In the **Forget Value** dropdown, select the number of days (**1**-**30 days**) after which the value is forgotten.

layouts/shortcodes/mdoc/en/security/cloud_siem/group_signals.mdoc.md

@@ -0,0 +1 @@
+Toggle **Group signals** if you want to reduce the number of signals generated. Then, select one or more groups for which you want to generate one security signal each.

layouts/shortcodes/mdoc/en/security/cloud_siem/impossible_travel_query.mdoc.md

@@ -0,0 +1,10 @@
+1. In the **User attribute** dropdown menu, select the log attribute that contains the user ID. This can be an identifier like an email address, user name, or account identifier.
+1. The **Location attribute** value is automatically set to `@network.client.geoip`.
+    - The `location attribute` specifies which field holds the geographic information for a log.
+    - The only supported value is `@network.client.geoip`, which is enriched by the [GeoIP parser][801] to give a log location information based on the client's IP address.
+1. Select the **Baseline user locations** checkbox if you want Datadog to learn regular access locations before triggering a signal.
+    - When selected, signals are suppressed for the first 24 hours. During that time, Datadog learns the user's regular access locations. This can be helpful to reduce noise and infer VPN usage or credentialed API access.
+    - See [How the impossible detection method works][802] for more information.
+
+[801]: /logs/log_configuration/processors/?tab=ui#geoip-parser
+[802]: /security/cloud_siem/detect_and_monitor/custom_detection_rules/impossible_travel/#how-the-impossible-travel-method-works

layouts/shortcodes/mdoc/en/security/cloud_siem/job_multi_triggering.mdoc.md

@@ -0,0 +1,7 @@
+In the **Job multi-triggering behavior** section, configure how often to keep updating the same signal when new values are detected within a specified time frame. For example, the same signal updates when any new value is detected within 1 hour, for a maximum duration of 24 hours.
+- An `evaluation window` defines a sliding period in which at least one case evaluates as true and assesses cases in real time.
+- After a signal is generated, the signal remains "open" if a case is matched at least once within the `keep alive` window. Each time a new event matches any of the cases, the *last updated* timestamp is updated for the signal.
+- A signal closes after the time exceeds the `maximum signal duration`, regardless of the query being matched. This time is calculated from the first seen timestamp.
+{% alert level="info" %}
+The `evaluation window` must be less than or equal to the `keep alive` and `maximum signal duration`.
+{% /alert %}

layouts/shortcodes/mdoc/en/security/cloud_siem/new_value_query.mdoc.md

@@ -0,0 +1,16 @@
+1. In the **Detect new value** dropdown menu, select the attributes you want to detect.
+    - For example, you can create a query for successful user authentication with the following settings:
+        - **Detect new value** is `country`
+        - **group by** is `user`
+        - Learning duration is `after 7 days`
+      {% br /%}Then, logs coming in over the next 7 days are evaluated with those configured values. If a log comes in with a new value after the learning duration (`7 days`), a signal is generated, and the new value is learned to prevent future signals with this value.
+    - You can also identify users and entities using multiple **Detect new value** attributes in a single query.
+        - For example, if you want to detect when a user signs in from a new device and from a country that they've never signed in from before, add `device_id` and `country_name` to the **Detect new value** field.
+1. (Optional) Define a signal grouping in the **group by** dropdown menu.
+    - The defined `group by` generates a signal for each `group by` value.
+    - Typically, the `group by` is an entity (like user or IP address).
+1. In the dropdown menu to the right of **group by**, select the learning duration.
+1. (Optional) Define a signal grouping in the **group by** dropdown menu.
+    - The defined `group by` generates a signal for each `group by` value.
+    - Typically, the `group by` is an entity (like user or IP address).
+1. In the dropdown menu to the right of **group by**, select the learning duration.

layouts/shortcodes/mdoc/en/security/cloud_siem/rule_multi_triggering.mdoc.md

@@ -0,0 +1,7 @@
+Configure how often you want to keep updating the same signal if new values are detected within a specified time frame. For example, the same signal updates if any new value is detected within 1 hour, for a maximum duration of 24 hours.
+- An `evaluation window` is specified to match when at least one of the cases matches true. This is a sliding window and evaluates cases in real time.
+- After a signal is generated, the signal remains "open" if a case is matched at least once within the `keep alive` window. Each time a new event matches any of the cases, the *last updated* timestamp is updated for the signal.
+- A signal closes after the time exceeds the `maximum signal duration`, regardless of the query being matched. This time is calculated from the first seen timestamp.
+{% alert level="info" %}
+The `evaluation window` must be less than or equal to the `keep alive` and `maximum signal duration`.
+{% /alert %}

layouts/shortcodes/mdoc/en/security/cloud_siem/rule_multi_triggering_content_anomaly.mdoc.md

@@ -0,0 +1,6 @@
+Configure how often you want to keep updating the same signal if new values are detected within a specified time frame. For example, the same signal updates if any new value is detected within 1 hour, for a maximum duration of 24 hours.
+- After a signal is generated, the signal remains "open" if a case is matched at least once within the `keep alive` window. Each time a new event matches any of the cases, the *last updated* timestamp is updated for the signal.
+- A signal closes after the time exceeds the `maximum signal duration`, regardless of the query being matched. This time is calculated from the first seen timestamp.
+{% alert level="info" %}
+The `evaluation window` must be less than or equal to the `keep alive` and `maximum signal duration`.
+{% /alert %}

layouts/shortcodes/mdoc/en/security/cloud_siem/set_conditions_anomaly.mdoc.md

@@ -0,0 +1,8 @@
+1. In the **Set severity to** dropdown menu, select the appropriate severity level (`INFO`, `LOW`, `MEDIUM`, `HIGH`, `CRITICAL`).
+1. In the **Anomaly Percentile** dropdown menu, select a minimum percentage required for Cloud SIEM to generate a signal.{% br /%}The anomaly percentile refers to the log volume over the selected time period to your historical log volumes. If you select 99.5%, then Cloud SIEM only generates a signal if the number of logs is greater than 99.5% of all prior periods.
+1. (Optional) In the **And notify** section, click **Add Recipient** to configure [notification targets][101].
+    - You can create [notification rules][102] to manage notifications automatically, avoiding manual edits for each detection rule.
+
+<!-- Links shared with set_conditions.md -->
+[101]: /security_platform/notifications/#notification-channels
+[102]: /security/notifications/rules/

layouts/shortcodes/mdoc/en/security/cloud_siem/set_conditions_content_anomaly.mdoc.md

@@ -0,0 +1,17 @@
+1. (Optional) Click the pencil icon next to **Condition 1** if you want to rename the condition. This name is appended to the rule name when a signal is generated.
+1. In the **Set severity to** dropdown menu, select the appropriate severity level (`INFO`, `LOW`, `MEDIUM`, `HIGH`, `CRITICAL`).
+1. In the **Anomaly count** field, enter the condition for how many anomalous logs within the specified window are required to trigger a signal.
+    - For example, if the condition is `a >= 3` where `a` is the query, a signal is triggered if there are at least three anomalous logs within the evaluation window.
+    - All rule conditions are evaluated as condition statements. Thus, the order of the conditions affects which notifications are sent because the first condition to match generates the signal. Click and drag your rule conditions to change their ordering.
+    - A rule condition contains logical operations (`>`, `>=`, `&&`, `||`) to determine if a signal should be generated based on the event counts in the previously defined queries.
+    - The ASCII lowercase query labels are referenced in this section. An example rule condition for query `a` is `a > 3`.
+      {% alert level="info" %}
+      The query label must precede the operator. For example, `a > 3` is allowed; `3 < a` is not allowed.
+      {% /alert %}
+1. In the **within a window of** dropdown menu, select the time period during which a signal is triggered if the condition is met.
+    - An `evaluation window` is specified to match when at least one of the cases matches true. This is a sliding window and evaluates cases in real time.
+1. In the **And notify** section, click **Add Recipient** to optionally configure [notification targets][101].
+    - You can also create [notification rules][102] to avoid manual edits to notification preferences for individual detection rules.
+
+[101]: /security_platform/notifications/#notification-channels
+[102]: /security/notifications/rules/

layouts/shortcodes/mdoc/en/security/cloud_siem/set_conditions_severity_notify_only.mdoc.md

@@ -0,0 +1,7 @@
+1. In the **Set severity to** dropdown menu, select the appropriate severity level (`INFO`, `LOW`, `MEDIUM`, `HIGH`, `CRITICAL`).
+1. (Optional) In the **And notify** section, click **Add Recipient** to configure [notification targets][101].
+    - You can create [notification rules][102] to manage notifications automatically, avoiding manual edits for each detection rule.
+
+<!-- Links shared with set_conditions.md -->
+[101]: /security_platform/notifications/#notification-channels
+[102]: /security/notifications/rules/