diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index be7bf771e30e..aa3a8981b486 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -782,6 +782,14 @@ components: required: true schema: type: string + LLMObsEvalNamePathParameter: + description: The name of the custom LLM Observability evaluator configuration. + example: "my-custom-evaluator" + in: path + name: eval_name + required: true + schema: + type: string LLMObsExperimentIDPathParameter: description: The ID of the LLM Observability experiment. example: "3fd6b5e0-8910-4b1c-a7d0-5b84de329012" @@ -36781,6 +36789,418 @@ components: nullable: true type: string type: object + LLMObsCustomEvalConfigAssessmentCriteria: + description: Criteria used to assess the pass/fail result of a custom evaluator. + properties: + max_threshold: + description: Maximum numeric threshold for a passing result. + example: 1.0 + format: double + nullable: true + type: number + min_threshold: + description: Minimum numeric threshold for a passing result. + example: 0.7 + format: double + nullable: true + type: number + pass_values: + description: Specific output values considered as a passing result. + example: + - "pass" + - "yes" + items: + description: A value considered as a passing result. + type: string + nullable: true + type: array + pass_when: + description: When true, a boolean output of true is treated as passing. + example: true + nullable: true + type: boolean + type: object + LLMObsCustomEvalConfigAttributes: + description: Attributes of a custom LLM Observability evaluator configuration. + properties: + category: + description: Category of the evaluator. + example: "Custom" + type: string + created_at: + description: Timestamp when the evaluator configuration was created. + example: "2024-01-15T10:30:00Z" + format: date-time + type: string + created_by: + $ref: "#/components/schemas/LLMObsCustomEvalConfigUser" + eval_name: + description: Name of the custom evaluator. + example: "my-custom-evaluator" + type: string + last_updated_by: + $ref: "#/components/schemas/LLMObsCustomEvalConfigUser" + llm_judge_config: + $ref: "#/components/schemas/LLMObsCustomEvalConfigLLMJudgeConfig" + llm_provider: + $ref: "#/components/schemas/LLMObsCustomEvalConfigLLMProvider" + target: + $ref: "#/components/schemas/LLMObsCustomEvalConfigTarget" + updated_at: + description: Timestamp when the evaluator configuration was last updated. + example: "2024-01-15T10:30:00Z" + format: date-time + type: string + required: + - eval_name + - created_at + - updated_at + type: object + LLMObsCustomEvalConfigBedrockOptions: + description: AWS Bedrock-specific options for LLM provider configuration. + properties: + region: + description: AWS region for Bedrock. + example: "us-east-1" + type: string + type: object + LLMObsCustomEvalConfigData: + description: Data object for a custom LLM Observability evaluator configuration. + properties: + attributes: + $ref: "#/components/schemas/LLMObsCustomEvalConfigAttributes" + id: + description: Unique name identifier of the evaluator configuration. + example: "my-custom-evaluator" + type: string + type: + $ref: "#/components/schemas/LLMObsCustomEvalConfigType" + required: + - id + - type + - attributes + type: object + LLMObsCustomEvalConfigEvalScope: + description: Scope at which to evaluate spans. + enum: + - span + - trace + - session + example: "span" + type: string + x-enum-varnames: + - SPAN + - TRACE + - SESSION + LLMObsCustomEvalConfigInferenceParams: + description: LLM inference parameters for a custom evaluator. + properties: + frequency_penalty: + description: Frequency penalty to reduce repetition. + example: 0.0 + format: float + type: number + max_tokens: + description: Maximum number of tokens to generate. + example: 1024 + format: int64 + type: integer + presence_penalty: + description: Presence penalty to reduce repetition. + example: 0.0 + format: float + type: number + temperature: + description: Sampling temperature for the LLM. + example: 0.7 + format: float + type: number + top_k: + description: Top-k sampling parameter. + example: 50 + format: int64 + type: integer + top_p: + description: Top-p (nucleus) sampling parameter. + example: 1.0 + format: float + type: number + type: object + LLMObsCustomEvalConfigIntegrationProvider: + description: Name of the LLM integration provider. + enum: + - openai + - amazon-bedrock + - anthropic + - azure-openai + - vertex-ai + - llm-proxy + example: "openai" + type: string + x-enum-varnames: + - OPENAI + - AMAZON_BEDROCK + - ANTHROPIC + - AZURE_OPENAI + - VERTEX_AI + - LLM_PROXY + LLMObsCustomEvalConfigLLMJudgeConfig: + description: LLM judge configuration for a custom evaluator. + properties: + assessment_criteria: + $ref: "#/components/schemas/LLMObsCustomEvalConfigAssessmentCriteria" + inference_params: + $ref: "#/components/schemas/LLMObsCustomEvalConfigInferenceParams" + last_used_library_prompt_template_name: + description: Name of the last library prompt template used. + example: "sentiment-analysis-v1" + nullable: true + type: string + modified_library_prompt_template: + description: Whether the library prompt template was modified. + example: false + nullable: true + type: boolean + output_schema: + additionalProperties: {} + description: JSON schema describing the expected output format of the LLM judge. + nullable: true + type: object + parsing_type: + $ref: "#/components/schemas/LLMObsCustomEvalConfigParsingType" + prompt_template: + description: List of messages forming the LLM judge prompt template. + items: + $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptMessage" + type: array + required: + - inference_params + type: object + LLMObsCustomEvalConfigLLMProvider: + description: LLM provider configuration for a custom evaluator. + properties: + bedrock: + $ref: "#/components/schemas/LLMObsCustomEvalConfigBedrockOptions" + integration_account_id: + description: Integration account identifier. + example: "my-account-id" + type: string + integration_provider: + $ref: "#/components/schemas/LLMObsCustomEvalConfigIntegrationProvider" + model_name: + description: Name of the LLM model. + example: "gpt-4o" + type: string + vertex_ai: + $ref: "#/components/schemas/LLMObsCustomEvalConfigVertexAIOptions" + type: object + LLMObsCustomEvalConfigParsingType: + description: Output parsing type for a custom LLM judge evaluator. + enum: + - structured_output + - json + example: "structured_output" + type: string + x-enum-varnames: + - STRUCTURED_OUTPUT + - JSON + LLMObsCustomEvalConfigPromptContent: + description: A content block within a prompt message. + properties: + type: + description: Content block type. + example: "text" + type: string + value: + $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptContentValue" + required: + - type + - value + type: object + LLMObsCustomEvalConfigPromptContentValue: + description: Value of a prompt message content block. + properties: + text: + description: Text content of the message block. + example: "What is the sentiment of this review?" + type: string + tool_call: + $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptToolCall" + tool_call_result: + $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptToolResult" + type: object + LLMObsCustomEvalConfigPromptMessage: + description: A message in the prompt template for a custom LLM judge evaluator. + properties: + content: + description: Text content of the message. + example: "Rate the quality of the following response:" + type: string + contents: + description: Multi-part content blocks for the message. + items: + $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptContent" + type: array + role: + description: Role of the message author. + example: "user" + type: string + required: + - role + type: object + LLMObsCustomEvalConfigPromptToolCall: + description: A tool call within a prompt message. + properties: + arguments: + description: JSON-encoded arguments for the tool call. + example: '{"location": "San Francisco"}' + type: string + id: + description: Unique identifier of the tool call. + example: "call_abc123" + type: string + name: + description: Name of the tool being called. + example: "get_weather" + type: string + type: + description: Type of the tool call. + example: "function" + type: string + type: object + LLMObsCustomEvalConfigPromptToolResult: + description: A tool call result within a prompt message. + properties: + name: + description: Name of the tool that produced this result. + example: "get_weather" + type: string + result: + description: The result returned by the tool. + example: "sunny, 72F" + type: string + tool_id: + description: Identifier of the tool call this result corresponds to. + example: "call_abc123" + type: string + type: + description: Type of the tool result. + example: "function" + type: string + type: object + LLMObsCustomEvalConfigResponse: + description: Response containing a custom LLM Observability evaluator configuration. + properties: + data: + $ref: "#/components/schemas/LLMObsCustomEvalConfigData" + required: + - data + type: object + LLMObsCustomEvalConfigTarget: + description: Target application configuration for a custom evaluator. + properties: + application_name: + description: Name of the ML application this evaluator targets. + example: "my-llm-app" + type: string + enabled: + description: Whether the evaluator is active for the target application. + example: true + type: boolean + eval_scope: + $ref: "#/components/schemas/LLMObsCustomEvalConfigEvalScope" + nullable: true + filter: + description: Filter expression to select which spans to evaluate. + example: "@service:my-service" + nullable: true + type: string + root_spans_only: + description: When true, only root spans are evaluated. + example: true + nullable: true + type: boolean + sampling_percentage: + description: Percentage of traces to evaluate. Must be greater than 0 and at most 100. + example: 50.0 + format: float + nullable: true + type: number + required: + - application_name + - enabled + type: object + LLMObsCustomEvalConfigType: + description: Type of the custom LLM Observability evaluator configuration resource. + enum: + - evaluator_config + example: "evaluator_config" + type: string + x-enum-varnames: + - EVALUATOR_CONFIG + LLMObsCustomEvalConfigUpdateAttributes: + description: Attributes for creating or updating a custom LLM Observability evaluator configuration. + properties: + category: + description: Category of the evaluator. + example: "Custom" + type: string + eval_name: + description: Name of the custom evaluator. If provided, must match the eval_name path parameter. + example: "my-custom-evaluator" + type: string + llm_judge_config: + $ref: "#/components/schemas/LLMObsCustomEvalConfigLLMJudgeConfig" + llm_provider: + $ref: "#/components/schemas/LLMObsCustomEvalConfigLLMProvider" + target: + $ref: "#/components/schemas/LLMObsCustomEvalConfigTarget" + required: + - target + type: object + LLMObsCustomEvalConfigUpdateData: + description: Data object for creating or updating a custom LLM Observability evaluator configuration. + properties: + attributes: + $ref: "#/components/schemas/LLMObsCustomEvalConfigUpdateAttributes" + id: + description: Name of the evaluator. If provided, must match the eval_name path parameter. + example: "my-custom-evaluator" + type: string + type: + $ref: "#/components/schemas/LLMObsCustomEvalConfigType" + required: + - type + - attributes + type: object + LLMObsCustomEvalConfigUpdateRequest: + description: Request to create or update a custom LLM Observability evaluator configuration. + properties: + data: + $ref: "#/components/schemas/LLMObsCustomEvalConfigUpdateData" + required: + - data + type: object + LLMObsCustomEvalConfigUser: + description: A Datadog user associated with a custom evaluator configuration. + properties: + email: + description: Email address of the user. + example: "user@example.com" + type: string + type: object + LLMObsCustomEvalConfigVertexAIOptions: + description: Google Vertex AI-specific options for LLM provider configuration. + properties: + location: + description: Google Cloud region. + example: "us-central1" + type: string + project: + description: Google Cloud project ID. + example: "my-gcp-project" + type: string + type: object LLMObsDatasetDataAttributesRequest: description: Attributes for creating an LLM Observability dataset. properties: @@ -78603,6 +79023,199 @@ paths: x-unstable: |- This endpoint is in Preview and may introduce breaking changes. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/unstable/llm-obs/config/evaluators/custom/{eval_name}: + delete: + description: Delete a custom LLM Observability evaluator configuration by its name. + operationId: DeleteLLMObsCustomEvalConfig + parameters: + - $ref: "#/components/parameters/LLMObsEvalNamePathParameter" + responses: + "204": + description: No Content + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Delete a custom evaluator configuration + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + get: + description: Retrieve a custom LLM Observability evaluator configuration by its name. + operationId: GetLLMObsCustomEvalConfig + parameters: + - $ref: "#/components/parameters/LLMObsEvalNamePathParameter" + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + attributes: + category: "Custom" + created_at: "2024-01-15T10:30:00Z" + created_by: + email: "user@example.com" + eval_name: "my-custom-evaluator" + last_updated_by: + email: "user@example.com" + llm_judge_config: + inference_params: + max_tokens: 1024 + temperature: 0.7 + parsing_type: "structured_output" + llm_provider: + integration_provider: "openai" + model_name: "gpt-4o" + target: + application_name: "my-llm-app" + enabled: true + sampling_percentage: 50.0 + updated_at: "2024-01-15T10:30:00Z" + id: "my-custom-evaluator" + type: "evaluator_config" + schema: + $ref: "#/components/schemas/LLMObsCustomEvalConfigResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Get a custom evaluator configuration + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + put: + description: Create or update a custom LLM Observability evaluator configuration by its name. + operationId: UpdateLLMObsCustomEvalConfig + parameters: + - $ref: "#/components/parameters/LLMObsEvalNamePathParameter" + requestBody: + content: + application/json: + examples: + default: + value: + data: + attributes: + llm_judge_config: + inference_params: + max_tokens: 1024 + temperature: 0.7 + parsing_type: "structured_output" + llm_provider: + integration_provider: "openai" + model_name: "gpt-4o" + target: + application_name: "my-llm-app" + enabled: true + sampling_percentage: 50.0 + id: "my-custom-evaluator" + type: "evaluator_config" + schema: + $ref: "#/components/schemas/LLMObsCustomEvalConfigUpdateRequest" + description: Custom evaluator configuration payload. + required: true + responses: + "200": + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "422": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unprocessable Entity + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Create or update a custom evaluator configuration + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). /api/v2/actions-datastores: get: description: Lists all datastores for the organization. diff --git a/examples/v2/llm-observability/DeleteLLMObsCustomEvalConfig.rb b/examples/v2/llm-observability/DeleteLLMObsCustomEvalConfig.rb new file mode 100644 index 000000000000..2823d59aeb4f --- /dev/null +++ b/examples/v2/llm-observability/DeleteLLMObsCustomEvalConfig.rb @@ -0,0 +1,8 @@ +# Delete a custom evaluator configuration returns "No Content" response + +require "datadog_api_client" +DatadogAPIClient.configure do |config| + config.unstable_operations["v2.delete_llm_obs_custom_eval_config".to_sym] = true +end +api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new +api_instance.delete_llm_obs_custom_eval_config("eval_name") diff --git a/examples/v2/llm-observability/GetLLMObsCustomEvalConfig.rb b/examples/v2/llm-observability/GetLLMObsCustomEvalConfig.rb new file mode 100644 index 000000000000..f6b1c38a688a --- /dev/null +++ b/examples/v2/llm-observability/GetLLMObsCustomEvalConfig.rb @@ -0,0 +1,8 @@ +# Get a custom evaluator configuration returns "OK" response + +require "datadog_api_client" +DatadogAPIClient.configure do |config| + config.unstable_operations["v2.get_llm_obs_custom_eval_config".to_sym] = true +end +api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new +p api_instance.get_llm_obs_custom_eval_config("eval_name") diff --git a/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.rb b/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.rb new file mode 100644 index 000000000000..432cc216e9e1 --- /dev/null +++ b/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.rb @@ -0,0 +1,88 @@ +# Create or update a custom evaluator configuration returns "OK" response + +require "datadog_api_client" +DatadogAPIClient.configure do |config| + config.unstable_operations["v2.update_llm_obs_custom_eval_config".to_sym] = true +end +api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new + +body = DatadogAPIClient::V2::LLMObsCustomEvalConfigUpdateRequest.new({ + data: DatadogAPIClient::V2::LLMObsCustomEvalConfigUpdateData.new({ + attributes: DatadogAPIClient::V2::LLMObsCustomEvalConfigUpdateAttributes.new({ + category: "Custom", + eval_name: "my-custom-evaluator", + llm_judge_config: DatadogAPIClient::V2::LLMObsCustomEvalConfigLLMJudgeConfig.new({ + assessment_criteria: DatadogAPIClient::V2::LLMObsCustomEvalConfigAssessmentCriteria.new({ + max_threshold: 1.0, + min_threshold: 0.7, + pass_values: [ + "pass", + "yes", + ], + pass_when: true, + }), + inference_params: DatadogAPIClient::V2::LLMObsCustomEvalConfigInferenceParams.new({ + frequency_penalty: 0.0, + max_tokens: 1024, + presence_penalty: 0.0, + temperature: 0.7, + top_k: 50, + top_p: 1.0, + }), + last_used_library_prompt_template_name: "sentiment-analysis-v1", + modified_library_prompt_template: false, + output_schema: nil, + parsing_type: DatadogAPIClient::V2::LLMObsCustomEvalConfigParsingType::STRUCTURED_OUTPUT, + prompt_template: [ + DatadogAPIClient::V2::LLMObsCustomEvalConfigPromptMessage.new({ + content: "Rate the quality of the following response:", + contents: [ + DatadogAPIClient::V2::LLMObsCustomEvalConfigPromptContent.new({ + type: "text", + value: DatadogAPIClient::V2::LLMObsCustomEvalConfigPromptContentValue.new({ + text: "What is the sentiment of this review?", + tool_call: DatadogAPIClient::V2::LLMObsCustomEvalConfigPromptToolCall.new({ + arguments: '{"location": "San Francisco"}', + id: "call_abc123", + name: "get_weather", + type: "function", + }), + tool_call_result: DatadogAPIClient::V2::LLMObsCustomEvalConfigPromptToolResult.new({ + name: "get_weather", + result: "sunny, 72F", + tool_id: "call_abc123", + type: "function", + }), + }), + }), + ], + role: "user", + }), + ], + }), + llm_provider: DatadogAPIClient::V2::LLMObsCustomEvalConfigLLMProvider.new({ + bedrock: DatadogAPIClient::V2::LLMObsCustomEvalConfigBedrockOptions.new({ + region: "us-east-1", + }), + integration_account_id: "my-account-id", + integration_provider: DatadogAPIClient::V2::LLMObsCustomEvalConfigIntegrationProvider::OPENAI, + model_name: "gpt-4o", + vertex_ai: DatadogAPIClient::V2::LLMObsCustomEvalConfigVertexAIOptions.new({ + location: "us-central1", + project: "my-gcp-project", + }), + }), + target: DatadogAPIClient::V2::LLMObsCustomEvalConfigTarget.new({ + application_name: "my-llm-app", + enabled: true, + eval_scope: DatadogAPIClient::V2::LLMObsCustomEvalConfigEvalScope::SPAN, + filter: "@service:my-service", + root_spans_only: true, + sampling_percentage: 50.0, + }), + }), + id: "my-custom-evaluator", + type: DatadogAPIClient::V2::LLMObsCustomEvalConfigType::EVALUATOR_CONFIG, + }), +}) +p api_instance.update_llm_obs_custom_eval_config("eval_name", body) diff --git a/features/scenarios_model_mapping.rb b/features/scenarios_model_mapping.rb index ec581ec55b63..8bc60cde9f80 100644 --- a/features/scenarios_model_mapping.rb +++ b/features/scenarios_model_mapping.rb @@ -884,6 +884,120 @@ "v2.TriggerFleetSchedule" => { "id" => "String", }, + "v2.DeleteLLMObsCustomEvalConfig" => { + "eval_name" => "String", + }, + "v2.GetLLMObsCustomEvalConfig" => { + "eval_name" => "String", + }, + "v2.UpdateLLMObsCustomEvalConfig" => { + "eval_name" => "String", + "body" => "LLMObsCustomEvalConfigUpdateRequest", + }, + "v2.ListLLMObsAnnotationQueues" => { + "project_id" => "String", + "queue_ids" => "Array", + }, + "v2.CreateLLMObsAnnotationQueue" => { + "body" => "LLMObsAnnotationQueueRequest", + }, + "v2.DeleteLLMObsAnnotationQueue" => { + "queue_id" => "String", + }, + "v2.UpdateLLMObsAnnotationQueue" => { + "queue_id" => "String", + "body" => "LLMObsAnnotationQueueUpdateRequest", + }, + "v2.GetLLMObsAnnotatedInteractions" => { + "queue_id" => "String", + }, + "v2.CreateLLMObsAnnotationQueueInteractions" => { + "queue_id" => "String", + "body" => "LLMObsAnnotationQueueInteractionsRequest", + }, + "v2.DeleteLLMObsAnnotationQueueInteractions" => { + "queue_id" => "String", + "body" => "LLMObsDeleteAnnotationQueueInteractionsRequest", + }, + "v2.ListLLMObsExperiments" => { + "filter_project_id" => "String", + "filter_dataset_id" => "String", + "filter_id" => "String", + "page_cursor" => "String", + "page_limit" => "Integer", + }, + "v2.CreateLLMObsExperiment" => { + "body" => "LLMObsExperimentRequest", + }, + "v2.DeleteLLMObsExperiments" => { + "body" => "LLMObsDeleteExperimentsRequest", + }, + "v2.UpdateLLMObsExperiment" => { + "experiment_id" => "String", + "body" => "LLMObsExperimentUpdateRequest", + }, + "v2.CreateLLMObsExperimentEvents" => { + "experiment_id" => "String", + "body" => "LLMObsExperimentEventsRequest", + }, + "v2.ListLLMObsProjects" => { + "filter_id" => "String", + "filter_name" => "String", + "page_cursor" => "String", + "page_limit" => "Integer", + }, + "v2.CreateLLMObsProject" => { + "body" => "LLMObsProjectRequest", + }, + "v2.DeleteLLMObsProjects" => { + "body" => "LLMObsDeleteProjectsRequest", + }, + "v2.UpdateLLMObsProject" => { + "project_id" => "String", + "body" => "LLMObsProjectUpdateRequest", + }, + "v2.ListLLMObsDatasets" => { + "project_id" => "String", + "filter_name" => "String", + "filter_id" => "String", + "page_cursor" => "String", + "page_limit" => "Integer", + }, + "v2.CreateLLMObsDataset" => { + "project_id" => "String", + "body" => "LLMObsDatasetRequest", + }, + "v2.DeleteLLMObsDatasets" => { + "project_id" => "String", + "body" => "LLMObsDeleteDatasetsRequest", + }, + "v2.UpdateLLMObsDataset" => { + "project_id" => "String", + "dataset_id" => "String", + "body" => "LLMObsDatasetUpdateRequest", + }, + "v2.ListLLMObsDatasetRecords" => { + "project_id" => "String", + "dataset_id" => "String", + "filter_version" => "Integer", + "page_cursor" => "String", + "page_limit" => "Integer", + }, + "v2.UpdateLLMObsDatasetRecords" => { + "project_id" => "String", + "dataset_id" => "String", + "body" => "LLMObsDatasetRecordsUpdateRequest", + }, + "v2.CreateLLMObsDatasetRecords" => { + "project_id" => "String", + "dataset_id" => "String", + "body" => "LLMObsDatasetRecordsRequest", + }, + "v2.DeleteLLMObsDatasetRecords" => { + "project_id" => "String", + "dataset_id" => "String", + "body" => "LLMObsDeleteDatasetRecordsRequest", + }, "v2.CreateDatastore" => { "body" => "CreateAppsDatastoreRequest", }, @@ -2785,110 +2899,6 @@ "v2.UpdateIPAllowlist" => { "body" => "IPAllowlistUpdateRequest", }, - "v2.ListLLMObsAnnotationQueues" => { - "project_id" => "String", - "queue_ids" => "Array", - }, - "v2.CreateLLMObsAnnotationQueue" => { - "body" => "LLMObsAnnotationQueueRequest", - }, - "v2.DeleteLLMObsAnnotationQueue" => { - "queue_id" => "String", - }, - "v2.UpdateLLMObsAnnotationQueue" => { - "queue_id" => "String", - "body" => "LLMObsAnnotationQueueUpdateRequest", - }, - "v2.GetLLMObsAnnotatedInteractions" => { - "queue_id" => "String", - }, - "v2.CreateLLMObsAnnotationQueueInteractions" => { - "queue_id" => "String", - "body" => "LLMObsAnnotationQueueInteractionsRequest", - }, - "v2.DeleteLLMObsAnnotationQueueInteractions" => { - "queue_id" => "String", - "body" => "LLMObsDeleteAnnotationQueueInteractionsRequest", - }, - "v2.ListLLMObsExperiments" => { - "filter_project_id" => "String", - "filter_dataset_id" => "String", - "filter_id" => "String", - "page_cursor" => "String", - "page_limit" => "Integer", - }, - "v2.CreateLLMObsExperiment" => { - "body" => "LLMObsExperimentRequest", - }, - "v2.DeleteLLMObsExperiments" => { - "body" => "LLMObsDeleteExperimentsRequest", - }, - "v2.UpdateLLMObsExperiment" => { - "experiment_id" => "String", - "body" => "LLMObsExperimentUpdateRequest", - }, - "v2.CreateLLMObsExperimentEvents" => { - "experiment_id" => "String", - "body" => "LLMObsExperimentEventsRequest", - }, - "v2.ListLLMObsProjects" => { - "filter_id" => "String", - "filter_name" => "String", - "page_cursor" => "String", - "page_limit" => "Integer", - }, - "v2.CreateLLMObsProject" => { - "body" => "LLMObsProjectRequest", - }, - "v2.DeleteLLMObsProjects" => { - "body" => "LLMObsDeleteProjectsRequest", - }, - "v2.UpdateLLMObsProject" => { - "project_id" => "String", - "body" => "LLMObsProjectUpdateRequest", - }, - "v2.ListLLMObsDatasets" => { - "project_id" => "String", - "filter_name" => "String", - "filter_id" => "String", - "page_cursor" => "String", - "page_limit" => "Integer", - }, - "v2.CreateLLMObsDataset" => { - "project_id" => "String", - "body" => "LLMObsDatasetRequest", - }, - "v2.DeleteLLMObsDatasets" => { - "project_id" => "String", - "body" => "LLMObsDeleteDatasetsRequest", - }, - "v2.UpdateLLMObsDataset" => { - "project_id" => "String", - "dataset_id" => "String", - "body" => "LLMObsDatasetUpdateRequest", - }, - "v2.ListLLMObsDatasetRecords" => { - "project_id" => "String", - "dataset_id" => "String", - "filter_version" => "Integer", - "page_cursor" => "String", - "page_limit" => "Integer", - }, - "v2.UpdateLLMObsDatasetRecords" => { - "project_id" => "String", - "dataset_id" => "String", - "body" => "LLMObsDatasetRecordsUpdateRequest", - }, - "v2.CreateLLMObsDatasetRecords" => { - "project_id" => "String", - "dataset_id" => "String", - "body" => "LLMObsDatasetRecordsRequest", - }, - "v2.DeleteLLMObsDatasetRecords" => { - "project_id" => "String", - "dataset_id" => "String", - "body" => "LLMObsDeleteDatasetRecordsRequest", - }, "v2.SubmitLog" => { "content_encoding" => "ContentEncoding", "ddtags" => "String", diff --git a/features/v2/llm_observability.feature b/features/v2/llm_observability.feature index a39062899886..dc0a15364b94 100644 --- a/features/v2/llm_observability.feature +++ b/features/v2/llm_observability.feature @@ -175,6 +175,42 @@ Feature: LLM Observability When the request is sent Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability + Scenario: Create or update a custom evaluator configuration returns "Bad Request" response + Given operation "UpdateLLMObsCustomEvalConfig" enabled + And new "UpdateLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Create or update a custom evaluator configuration returns "Not Found" response + Given operation "UpdateLLMObsCustomEvalConfig" enabled + And new "UpdateLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: Create or update a custom evaluator configuration returns "OK" response + Given operation "UpdateLLMObsCustomEvalConfig" enabled + And new "UpdateLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + When the request is sent + Then the response status is 200 OK + + @generated @skip @team:DataDog/ml-observability + Scenario: Create or update a custom evaluator configuration returns "Unprocessable Entity" response + Given operation "UpdateLLMObsCustomEvalConfig" enabled + And new "UpdateLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + When the request is sent + Then the response status is 422 Unprocessable Entity + @generated @skip @team:DataDog/ml-observability Scenario: Delete LLM Observability dataset records returns "Bad Request" response Given operation "DeleteLLMObsDatasetRecords" enabled @@ -264,6 +300,30 @@ Feature: LLM Observability When the request is sent Then the response status is 204 No Content + @generated @skip @team:DataDog/ml-observability + Scenario: Delete a custom evaluator configuration returns "Bad Request" response + Given operation "DeleteLLMObsCustomEvalConfig" enabled + And new "DeleteLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Delete a custom evaluator configuration returns "No Content" response + Given operation "DeleteLLMObsCustomEvalConfig" enabled + And new "DeleteLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 204 No Content + + @generated @skip @team:DataDog/ml-observability + Scenario: Delete a custom evaluator configuration returns "Not Found" response + Given operation "DeleteLLMObsCustomEvalConfig" enabled + And new "DeleteLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 404 Not Found + @generated @skip @team:DataDog/ml-observability Scenario: Delete an LLM Observability annotation queue returns "No Content" response Given operation "DeleteLLMObsAnnotationQueue" enabled @@ -307,6 +367,30 @@ Feature: LLM Observability When the request is sent Then the response status is 404 Not Found + @generated @skip @team:DataDog/ml-observability + Scenario: Get a custom evaluator configuration returns "Bad Request" response + Given operation "GetLLMObsCustomEvalConfig" enabled + And new "GetLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Get a custom evaluator configuration returns "Not Found" response + Given operation "GetLLMObsCustomEvalConfig" enabled + And new "GetLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: Get a custom evaluator configuration returns "OK" response + Given operation "GetLLMObsCustomEvalConfig" enabled + And new "GetLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: Get annotated queue interactions returns "Bad Request" response Given operation "GetLLMObsAnnotatedInteractions" enabled diff --git a/features/v2/undo.json b/features/v2/undo.json index 21caf3f3290a..774684b94814 100644 --- a/features/v2/undo.json +++ b/features/v2/undo.json @@ -111,6 +111,31 @@ "type": "unsafe" } }, + "DeleteLLMObsCustomEvalConfig": { + "tag": "LLM Observability", + "undo": { + "operationId": "UpdateLLMObsCustomEvalConfig", + "parameters": [ + { + "name": "eval_name", + "source": "path" + } + ], + "type": "unsafe" + } + }, + "GetLLMObsCustomEvalConfig": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, + "UpdateLLMObsCustomEvalConfig": { + "tag": "LLM Observability", + "undo": { + "type": "idempotent" + } + }, "ListDatastores": { "tag": "Actions Datastores", "undo": { diff --git a/lib/datadog_api_client/configuration.rb b/lib/datadog_api_client/configuration.rb index 73968a285cc2..4d7788dd823c 100644 --- a/lib/datadog_api_client/configuration.rb +++ b/lib/datadog_api_client/configuration.rb @@ -203,6 +203,33 @@ def initialize "v2.list_fleet_schedules": false, "v2.trigger_fleet_schedule": false, "v2.update_fleet_schedule": false, + "v2.create_llm_obs_annotation_queue": false, + "v2.create_llm_obs_annotation_queue_interactions": false, + "v2.create_llm_obs_dataset": false, + "v2.create_llm_obs_dataset_records": false, + "v2.create_llm_obs_experiment": false, + "v2.create_llm_obs_experiment_events": false, + "v2.create_llm_obs_project": false, + "v2.delete_llm_obs_annotation_queue": false, + "v2.delete_llm_obs_annotation_queue_interactions": false, + "v2.delete_llm_obs_custom_eval_config": false, + "v2.delete_llm_obs_dataset_records": false, + "v2.delete_llm_obs_datasets": false, + "v2.delete_llm_obs_experiments": false, + "v2.delete_llm_obs_projects": false, + "v2.get_llm_obs_annotated_interactions": false, + "v2.get_llm_obs_custom_eval_config": false, + "v2.list_llm_obs_annotation_queues": false, + "v2.list_llm_obs_dataset_records": false, + "v2.list_llm_obs_datasets": false, + "v2.list_llm_obs_experiments": false, + "v2.list_llm_obs_projects": false, + "v2.update_llm_obs_annotation_queue": false, + "v2.update_llm_obs_custom_eval_config": false, + "v2.update_llm_obs_dataset": false, + "v2.update_llm_obs_dataset_records": false, + "v2.update_llm_obs_experiment": false, + "v2.update_llm_obs_project": false, "v2.create_open_api": false, "v2.delete_open_api": false, "v2.get_open_api": false, @@ -351,30 +378,6 @@ def initialize "v2.update_jira_issue_template": false, "v2.create_tenancy_config": false, "v2.get_tenancy_configs": false, - "v2.create_llm_obs_annotation_queue": false, - "v2.create_llm_obs_annotation_queue_interactions": false, - "v2.create_llm_obs_dataset": false, - "v2.create_llm_obs_dataset_records": false, - "v2.create_llm_obs_experiment": false, - "v2.create_llm_obs_experiment_events": false, - "v2.create_llm_obs_project": false, - "v2.delete_llm_obs_annotation_queue": false, - "v2.delete_llm_obs_annotation_queue_interactions": false, - "v2.delete_llm_obs_dataset_records": false, - "v2.delete_llm_obs_datasets": false, - "v2.delete_llm_obs_experiments": false, - "v2.delete_llm_obs_projects": false, - "v2.get_llm_obs_annotated_interactions": false, - "v2.list_llm_obs_annotation_queues": false, - "v2.list_llm_obs_dataset_records": false, - "v2.list_llm_obs_datasets": false, - "v2.list_llm_obs_experiments": false, - "v2.list_llm_obs_projects": false, - "v2.update_llm_obs_annotation_queue": false, - "v2.update_llm_obs_dataset": false, - "v2.update_llm_obs_dataset_records": false, - "v2.update_llm_obs_experiment": false, - "v2.update_llm_obs_project": false, "v2.add_role_to_restriction_query": false, "v2.create_restriction_query": false, "v2.delete_restriction_query": false, diff --git a/lib/datadog_api_client/inflector.rb b/lib/datadog_api_client/inflector.rb index 6afe03aa604a..8a68954f1e56 100644 --- a/lib/datadog_api_client/inflector.rb +++ b/lib/datadog_api_client/inflector.rb @@ -3380,6 +3380,29 @@ def overrides "v2.llm_obs_annotation_queue_update_data_request" => "LLMObsAnnotationQueueUpdateDataRequest", "v2.llm_obs_annotation_queue_update_request" => "LLMObsAnnotationQueueUpdateRequest", "v2.llm_obs_cursor_meta" => "LLMObsCursorMeta", + "v2.llm_obs_custom_eval_config_assessment_criteria" => "LLMObsCustomEvalConfigAssessmentCriteria", + "v2.llm_obs_custom_eval_config_attributes" => "LLMObsCustomEvalConfigAttributes", + "v2.llm_obs_custom_eval_config_bedrock_options" => "LLMObsCustomEvalConfigBedrockOptions", + "v2.llm_obs_custom_eval_config_data" => "LLMObsCustomEvalConfigData", + "v2.llm_obs_custom_eval_config_eval_scope" => "LLMObsCustomEvalConfigEvalScope", + "v2.llm_obs_custom_eval_config_inference_params" => "LLMObsCustomEvalConfigInferenceParams", + "v2.llm_obs_custom_eval_config_integration_provider" => "LLMObsCustomEvalConfigIntegrationProvider", + "v2.llm_obs_custom_eval_config_llm_judge_config" => "LLMObsCustomEvalConfigLLMJudgeConfig", + "v2.llm_obs_custom_eval_config_llm_provider" => "LLMObsCustomEvalConfigLLMProvider", + "v2.llm_obs_custom_eval_config_parsing_type" => "LLMObsCustomEvalConfigParsingType", + "v2.llm_obs_custom_eval_config_prompt_content" => "LLMObsCustomEvalConfigPromptContent", + "v2.llm_obs_custom_eval_config_prompt_content_value" => "LLMObsCustomEvalConfigPromptContentValue", + "v2.llm_obs_custom_eval_config_prompt_message" => "LLMObsCustomEvalConfigPromptMessage", + "v2.llm_obs_custom_eval_config_prompt_tool_call" => "LLMObsCustomEvalConfigPromptToolCall", + "v2.llm_obs_custom_eval_config_prompt_tool_result" => "LLMObsCustomEvalConfigPromptToolResult", + "v2.llm_obs_custom_eval_config_response" => "LLMObsCustomEvalConfigResponse", + "v2.llm_obs_custom_eval_config_target" => "LLMObsCustomEvalConfigTarget", + "v2.llm_obs_custom_eval_config_type" => "LLMObsCustomEvalConfigType", + "v2.llm_obs_custom_eval_config_update_attributes" => "LLMObsCustomEvalConfigUpdateAttributes", + "v2.llm_obs_custom_eval_config_update_data" => "LLMObsCustomEvalConfigUpdateData", + "v2.llm_obs_custom_eval_config_update_request" => "LLMObsCustomEvalConfigUpdateRequest", + "v2.llm_obs_custom_eval_config_user" => "LLMObsCustomEvalConfigUser", + "v2.llm_obs_custom_eval_config_vertex_ai_options" => "LLMObsCustomEvalConfigVertexAIOptions", "v2.llm_obs_dataset_data_attributes_request" => "LLMObsDatasetDataAttributesRequest", "v2.llm_obs_dataset_data_attributes_response" => "LLMObsDatasetDataAttributesResponse", "v2.llm_obs_dataset_data_request" => "LLMObsDatasetDataRequest", diff --git a/lib/datadog_api_client/v2/api/llm_observability_api.rb b/lib/datadog_api_client/v2/api/llm_observability_api.rb index 714eb6cc6648..c41159b32fc8 100644 --- a/lib/datadog_api_client/v2/api/llm_observability_api.rb +++ b/lib/datadog_api_client/v2/api/llm_observability_api.rb @@ -710,6 +710,77 @@ def delete_llm_obs_annotation_queue_interactions_with_http_info(queue_id, body, return data, status_code, headers end + # Delete a custom evaluator configuration. + # + # @see #delete_llm_obs_custom_eval_config_with_http_info + def delete_llm_obs_custom_eval_config(eval_name, opts = {}) + delete_llm_obs_custom_eval_config_with_http_info(eval_name, opts) + nil + end + + # Delete a custom evaluator configuration. + # + # Delete a custom LLM Observability evaluator configuration by its name. + # + # @param eval_name [String] The name of the custom LLM Observability evaluator configuration. + # @param opts [Hash] the optional parameters + # @return [Array<(nil, Integer, Hash)>] nil, response status code and response headers + def delete_llm_obs_custom_eval_config_with_http_info(eval_name, opts = {}) + unstable_enabled = @api_client.config.unstable_operations["v2.delete_llm_obs_custom_eval_config".to_sym] + if unstable_enabled + @api_client.config.logger.warn format("Using unstable operation '%s'", "v2.delete_llm_obs_custom_eval_config") + else + raise DatadogAPIClient::APIError.new(message: format("Unstable operation '%s' is disabled", "v2.delete_llm_obs_custom_eval_config")) + end + + if @api_client.config.debugging + @api_client.config.logger.debug 'Calling API: LLMObservabilityAPI.delete_llm_obs_custom_eval_config ...' + end + # verify the required parameter 'eval_name' is set + if @api_client.config.client_side_validation && eval_name.nil? + fail ArgumentError, "Missing the required parameter 'eval_name' when calling LLMObservabilityAPI.delete_llm_obs_custom_eval_config" + end + # resource path + local_var_path = '/api/unstable/llm-obs/config/evaluators/custom/{eval_name}'.sub('{eval_name}', CGI.escape(eval_name.to_s).gsub('%2F', '/')) + + # query parameters + query_params = opts[:query_params] || {} + + # header parameters + header_params = opts[:header_params] || {} + # HTTP header 'Accept' (if needed) + header_params['Accept'] = @api_client.select_header_accept(['*/*']) + + # form parameters + form_params = opts[:form_params] || {} + + # http body (model) + post_body = opts[:debug_body] + + # return_type + return_type = opts[:debug_return_type] + + # auth_names + auth_names = opts[:debug_auth_names] || [:apiKeyAuth, :appKeyAuth] + + new_options = opts.merge( + :operation => :delete_llm_obs_custom_eval_config, + :header_params => header_params, + :query_params => query_params, + :form_params => form_params, + :body => post_body, + :auth_names => auth_names, + :return_type => return_type, + :api_version => "V2" + ) + + data, status_code, headers = @api_client.call_api(Net::HTTP::Delete, local_var_path, new_options) + if @api_client.config.debugging + @api_client.config.logger.debug "API called: LLMObservabilityAPI#delete_llm_obs_custom_eval_config\nData: #{data.inspect}\nStatus code: #{status_code}\nHeaders: #{headers}" + end + return data, status_code, headers + end + # Delete LLM Observability dataset records. # # @see #delete_llm_obs_dataset_records_with_http_info @@ -1088,6 +1159,77 @@ def get_llm_obs_annotated_interactions_with_http_info(queue_id, opts = {}) return data, status_code, headers end + # Get a custom evaluator configuration. + # + # @see #get_llm_obs_custom_eval_config_with_http_info + def get_llm_obs_custom_eval_config(eval_name, opts = {}) + data, _status_code, _headers = get_llm_obs_custom_eval_config_with_http_info(eval_name, opts) + data + end + + # Get a custom evaluator configuration. + # + # Retrieve a custom LLM Observability evaluator configuration by its name. + # + # @param eval_name [String] The name of the custom LLM Observability evaluator configuration. + # @param opts [Hash] the optional parameters + # @return [Array<(LLMObsCustomEvalConfigResponse, Integer, Hash)>] LLMObsCustomEvalConfigResponse data, response status code and response headers + def get_llm_obs_custom_eval_config_with_http_info(eval_name, opts = {}) + unstable_enabled = @api_client.config.unstable_operations["v2.get_llm_obs_custom_eval_config".to_sym] + if unstable_enabled + @api_client.config.logger.warn format("Using unstable operation '%s'", "v2.get_llm_obs_custom_eval_config") + else + raise DatadogAPIClient::APIError.new(message: format("Unstable operation '%s' is disabled", "v2.get_llm_obs_custom_eval_config")) + end + + if @api_client.config.debugging + @api_client.config.logger.debug 'Calling API: LLMObservabilityAPI.get_llm_obs_custom_eval_config ...' + end + # verify the required parameter 'eval_name' is set + if @api_client.config.client_side_validation && eval_name.nil? + fail ArgumentError, "Missing the required parameter 'eval_name' when calling LLMObservabilityAPI.get_llm_obs_custom_eval_config" + end + # resource path + local_var_path = '/api/unstable/llm-obs/config/evaluators/custom/{eval_name}'.sub('{eval_name}', CGI.escape(eval_name.to_s).gsub('%2F', '/')) + + # query parameters + query_params = opts[:query_params] || {} + + # header parameters + header_params = opts[:header_params] || {} + # HTTP header 'Accept' (if needed) + header_params['Accept'] = @api_client.select_header_accept(['application/json']) + + # form parameters + form_params = opts[:form_params] || {} + + # http body (model) + post_body = opts[:debug_body] + + # return_type + return_type = opts[:debug_return_type] || 'LLMObsCustomEvalConfigResponse' + + # auth_names + auth_names = opts[:debug_auth_names] || [:apiKeyAuth, :appKeyAuth] + + new_options = opts.merge( + :operation => :get_llm_obs_custom_eval_config, + :header_params => header_params, + :query_params => query_params, + :form_params => form_params, + :body => post_body, + :auth_names => auth_names, + :return_type => return_type, + :api_version => "V2" + ) + + data, status_code, headers = @api_client.call_api(Net::HTTP::Get, local_var_path, new_options) + if @api_client.config.debugging + @api_client.config.logger.debug "API called: LLMObservabilityAPI#get_llm_obs_custom_eval_config\nData: #{data.inspect}\nStatus code: #{status_code}\nHeaders: #{headers}" + end + return data, status_code, headers + end + # List LLM Observability annotation queues. # # @see #list_llm_obs_annotation_queues_with_http_info @@ -1549,6 +1691,84 @@ def update_llm_obs_annotation_queue_with_http_info(queue_id, body, opts = {}) return data, status_code, headers end + # Create or update a custom evaluator configuration. + # + # @see #update_llm_obs_custom_eval_config_with_http_info + def update_llm_obs_custom_eval_config(eval_name, body, opts = {}) + update_llm_obs_custom_eval_config_with_http_info(eval_name, body, opts) + nil + end + + # Create or update a custom evaluator configuration. + # + # Create or update a custom LLM Observability evaluator configuration by its name. + # + # @param eval_name [String] The name of the custom LLM Observability evaluator configuration. + # @param body [LLMObsCustomEvalConfigUpdateRequest] Custom evaluator configuration payload. + # @param opts [Hash] the optional parameters + # @return [Array<(nil, Integer, Hash)>] nil, response status code and response headers + def update_llm_obs_custom_eval_config_with_http_info(eval_name, body, opts = {}) + unstable_enabled = @api_client.config.unstable_operations["v2.update_llm_obs_custom_eval_config".to_sym] + if unstable_enabled + @api_client.config.logger.warn format("Using unstable operation '%s'", "v2.update_llm_obs_custom_eval_config") + else + raise DatadogAPIClient::APIError.new(message: format("Unstable operation '%s' is disabled", "v2.update_llm_obs_custom_eval_config")) + end + + if @api_client.config.debugging + @api_client.config.logger.debug 'Calling API: LLMObservabilityAPI.update_llm_obs_custom_eval_config ...' + end + # verify the required parameter 'eval_name' is set + if @api_client.config.client_side_validation && eval_name.nil? + fail ArgumentError, "Missing the required parameter 'eval_name' when calling LLMObservabilityAPI.update_llm_obs_custom_eval_config" + end + # verify the required parameter 'body' is set + if @api_client.config.client_side_validation && body.nil? + fail ArgumentError, "Missing the required parameter 'body' when calling LLMObservabilityAPI.update_llm_obs_custom_eval_config" + end + # resource path + local_var_path = '/api/unstable/llm-obs/config/evaluators/custom/{eval_name}'.sub('{eval_name}', CGI.escape(eval_name.to_s).gsub('%2F', '/')) + + # query parameters + query_params = opts[:query_params] || {} + + # header parameters + header_params = opts[:header_params] || {} + # HTTP header 'Accept' (if needed) + header_params['Accept'] = @api_client.select_header_accept(['*/*']) + # HTTP header 'Content-Type' + header_params['Content-Type'] = @api_client.select_header_content_type(['application/json']) + + # form parameters + form_params = opts[:form_params] || {} + + # http body (model) + post_body = opts[:debug_body] || @api_client.object_to_http_body(body) + + # return_type + return_type = opts[:debug_return_type] + + # auth_names + auth_names = opts[:debug_auth_names] || [:apiKeyAuth, :appKeyAuth] + + new_options = opts.merge( + :operation => :update_llm_obs_custom_eval_config, + :header_params => header_params, + :query_params => query_params, + :form_params => form_params, + :body => post_body, + :auth_names => auth_names, + :return_type => return_type, + :api_version => "V2" + ) + + data, status_code, headers = @api_client.call_api(Net::HTTP::Put, local_var_path, new_options) + if @api_client.config.debugging + @api_client.config.logger.debug "API called: LLMObservabilityAPI#update_llm_obs_custom_eval_config\nData: #{data.inspect}\nStatus code: #{status_code}\nHeaders: #{headers}" + end + return data, status_code, headers + end + # Update an LLM Observability dataset. # # @see #update_llm_obs_dataset_with_http_info diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_assessment_criteria.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_assessment_criteria.rb new file mode 100644 index 000000000000..b4d5f7ef610f --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_assessment_criteria.rb @@ -0,0 +1,148 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Criteria used to assess the pass/fail result of a custom evaluator. + class LLMObsCustomEvalConfigAssessmentCriteria + include BaseGenericModel + + # Maximum numeric threshold for a passing result. + attr_accessor :max_threshold + + # Minimum numeric threshold for a passing result. + attr_accessor :min_threshold + + # Specific output values considered as a passing result. + attr_accessor :pass_values + + # When true, a boolean output of true is treated as passing. + attr_accessor :pass_when + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'max_threshold' => :'max_threshold', + :'min_threshold' => :'min_threshold', + :'pass_values' => :'pass_values', + :'pass_when' => :'pass_when' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'max_threshold' => :'Float', + :'min_threshold' => :'Float', + :'pass_values' => :'Array', + :'pass_when' => :'Boolean' + } + end + + # List of attributes with nullable: true + # @!visibility private + def self.openapi_nullable + Set.new([ + :'max_threshold', + :'min_threshold', + :'pass_values', + :'pass_when', + ]) + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigAssessmentCriteria` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'max_threshold') + self.max_threshold = attributes[:'max_threshold'] + end + + if attributes.key?(:'min_threshold') + self.min_threshold = attributes[:'min_threshold'] + end + + if attributes.key?(:'pass_values') + if (value = attributes[:'pass_values']).is_a?(Array) + self.pass_values = value + end + end + + if attributes.key?(:'pass_when') + self.pass_when = attributes[:'pass_when'] + end + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + max_threshold == o.max_threshold && + min_threshold == o.min_threshold && + pass_values == o.pass_values && + pass_when == o.pass_when && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [max_threshold, min_threshold, pass_values, pass_when, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_attributes.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_attributes.rb new file mode 100644 index 000000000000..0d9cb82d9bb8 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_attributes.rb @@ -0,0 +1,225 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Attributes of a custom LLM Observability evaluator configuration. + class LLMObsCustomEvalConfigAttributes + include BaseGenericModel + + # Category of the evaluator. + attr_accessor :category + + # Timestamp when the evaluator configuration was created. + attr_reader :created_at + + # A Datadog user associated with a custom evaluator configuration. + attr_accessor :created_by + + # Name of the custom evaluator. + attr_reader :eval_name + + # A Datadog user associated with a custom evaluator configuration. + attr_accessor :last_updated_by + + # LLM judge configuration for a custom evaluator. + attr_accessor :llm_judge_config + + # LLM provider configuration for a custom evaluator. + attr_accessor :llm_provider + + # Target application configuration for a custom evaluator. + attr_accessor :target + + # Timestamp when the evaluator configuration was last updated. + attr_reader :updated_at + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'category' => :'category', + :'created_at' => :'created_at', + :'created_by' => :'created_by', + :'eval_name' => :'eval_name', + :'last_updated_by' => :'last_updated_by', + :'llm_judge_config' => :'llm_judge_config', + :'llm_provider' => :'llm_provider', + :'target' => :'target', + :'updated_at' => :'updated_at' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'category' => :'String', + :'created_at' => :'Time', + :'created_by' => :'LLMObsCustomEvalConfigUser', + :'eval_name' => :'String', + :'last_updated_by' => :'LLMObsCustomEvalConfigUser', + :'llm_judge_config' => :'LLMObsCustomEvalConfigLLMJudgeConfig', + :'llm_provider' => :'LLMObsCustomEvalConfigLLMProvider', + :'target' => :'LLMObsCustomEvalConfigTarget', + :'updated_at' => :'Time' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigAttributes` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'category') + self.category = attributes[:'category'] + end + + if attributes.key?(:'created_at') + self.created_at = attributes[:'created_at'] + end + + if attributes.key?(:'created_by') + self.created_by = attributes[:'created_by'] + end + + if attributes.key?(:'eval_name') + self.eval_name = attributes[:'eval_name'] + end + + if attributes.key?(:'last_updated_by') + self.last_updated_by = attributes[:'last_updated_by'] + end + + if attributes.key?(:'llm_judge_config') + self.llm_judge_config = attributes[:'llm_judge_config'] + end + + if attributes.key?(:'llm_provider') + self.llm_provider = attributes[:'llm_provider'] + end + + if attributes.key?(:'target') + self.target = attributes[:'target'] + end + + if attributes.key?(:'updated_at') + self.updated_at = attributes[:'updated_at'] + end + end + + # Check to see if the all the properties in the model are valid + # @return true if the model is valid + # @!visibility private + def valid? + return false if @created_at.nil? + return false if @eval_name.nil? + return false if @updated_at.nil? + true + end + + # Custom attribute writer method with validation + # @param created_at [Object] Object to be assigned + # @!visibility private + def created_at=(created_at) + if created_at.nil? + fail ArgumentError, 'invalid value for "created_at", created_at cannot be nil.' + end + @created_at = created_at + end + + # Custom attribute writer method with validation + # @param eval_name [Object] Object to be assigned + # @!visibility private + def eval_name=(eval_name) + if eval_name.nil? + fail ArgumentError, 'invalid value for "eval_name", eval_name cannot be nil.' + end + @eval_name = eval_name + end + + # Custom attribute writer method with validation + # @param updated_at [Object] Object to be assigned + # @!visibility private + def updated_at=(updated_at) + if updated_at.nil? + fail ArgumentError, 'invalid value for "updated_at", updated_at cannot be nil.' + end + @updated_at = updated_at + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + category == o.category && + created_at == o.created_at && + created_by == o.created_by && + eval_name == o.eval_name && + last_updated_by == o.last_updated_by && + llm_judge_config == o.llm_judge_config && + llm_provider == o.llm_provider && + target == o.target && + updated_at == o.updated_at && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [category, created_at, created_by, eval_name, last_updated_by, llm_judge_config, llm_provider, target, updated_at, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_bedrock_options.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_bedrock_options.rb new file mode 100644 index 000000000000..261bed2ec872 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_bedrock_options.rb @@ -0,0 +1,105 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # AWS Bedrock-specific options for LLM provider configuration. + class LLMObsCustomEvalConfigBedrockOptions + include BaseGenericModel + + # AWS region for Bedrock. + attr_accessor :region + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'region' => :'region' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'region' => :'String' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigBedrockOptions` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'region') + self.region = attributes[:'region'] + end + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + region == o.region && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [region, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_data.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_data.rb new file mode 100644 index 000000000000..456fac6a3b76 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_data.rb @@ -0,0 +1,165 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Data object for a custom LLM Observability evaluator configuration. + class LLMObsCustomEvalConfigData + include BaseGenericModel + + # Attributes of a custom LLM Observability evaluator configuration. + attr_reader :attributes + + # Unique name identifier of the evaluator configuration. + attr_reader :id + + # Type of the custom LLM Observability evaluator configuration resource. + attr_reader :type + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'attributes' => :'attributes', + :'id' => :'id', + :'type' => :'type' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'attributes' => :'LLMObsCustomEvalConfigAttributes', + :'id' => :'String', + :'type' => :'LLMObsCustomEvalConfigType' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigData` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'attributes') + self.attributes = attributes[:'attributes'] + end + + if attributes.key?(:'id') + self.id = attributes[:'id'] + end + + if attributes.key?(:'type') + self.type = attributes[:'type'] + end + end + + # Check to see if the all the properties in the model are valid + # @return true if the model is valid + # @!visibility private + def valid? + return false if @attributes.nil? + return false if @id.nil? + return false if @type.nil? + true + end + + # Custom attribute writer method with validation + # @param attributes [Object] Object to be assigned + # @!visibility private + def attributes=(attributes) + if attributes.nil? + fail ArgumentError, 'invalid value for "attributes", attributes cannot be nil.' + end + @attributes = attributes + end + + # Custom attribute writer method with validation + # @param id [Object] Object to be assigned + # @!visibility private + def id=(id) + if id.nil? + fail ArgumentError, 'invalid value for "id", id cannot be nil.' + end + @id = id + end + + # Custom attribute writer method with validation + # @param type [Object] Object to be assigned + # @!visibility private + def type=(type) + if type.nil? + fail ArgumentError, 'invalid value for "type", type cannot be nil.' + end + @type = type + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + attributes == o.attributes && + id == o.id && + type == o.type && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [attributes, id, type, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_eval_scope.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_eval_scope.rb new file mode 100644 index 000000000000..cc980bd70b46 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_eval_scope.rb @@ -0,0 +1,28 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Scope at which to evaluate spans. + class LLMObsCustomEvalConfigEvalScope + include BaseEnumModel + + SPAN = "span".freeze + TRACE = "trace".freeze + SESSION = "session".freeze + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_inference_params.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_inference_params.rb new file mode 100644 index 000000000000..ce677ba2e6ee --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_inference_params.rb @@ -0,0 +1,155 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # LLM inference parameters for a custom evaluator. + class LLMObsCustomEvalConfigInferenceParams + include BaseGenericModel + + # Frequency penalty to reduce repetition. + attr_accessor :frequency_penalty + + # Maximum number of tokens to generate. + attr_accessor :max_tokens + + # Presence penalty to reduce repetition. + attr_accessor :presence_penalty + + # Sampling temperature for the LLM. + attr_accessor :temperature + + # Top-k sampling parameter. + attr_accessor :top_k + + # Top-p (nucleus) sampling parameter. + attr_accessor :top_p + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'frequency_penalty' => :'frequency_penalty', + :'max_tokens' => :'max_tokens', + :'presence_penalty' => :'presence_penalty', + :'temperature' => :'temperature', + :'top_k' => :'top_k', + :'top_p' => :'top_p' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'frequency_penalty' => :'Float', + :'max_tokens' => :'Integer', + :'presence_penalty' => :'Float', + :'temperature' => :'Float', + :'top_k' => :'Integer', + :'top_p' => :'Float' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigInferenceParams` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'frequency_penalty') + self.frequency_penalty = attributes[:'frequency_penalty'] + end + + if attributes.key?(:'max_tokens') + self.max_tokens = attributes[:'max_tokens'] + end + + if attributes.key?(:'presence_penalty') + self.presence_penalty = attributes[:'presence_penalty'] + end + + if attributes.key?(:'temperature') + self.temperature = attributes[:'temperature'] + end + + if attributes.key?(:'top_k') + self.top_k = attributes[:'top_k'] + end + + if attributes.key?(:'top_p') + self.top_p = attributes[:'top_p'] + end + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + frequency_penalty == o.frequency_penalty && + max_tokens == o.max_tokens && + presence_penalty == o.presence_penalty && + temperature == o.temperature && + top_k == o.top_k && + top_p == o.top_p && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [frequency_penalty, max_tokens, presence_penalty, temperature, top_k, top_p, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_integration_provider.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_integration_provider.rb new file mode 100644 index 000000000000..865b9a0a748e --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_integration_provider.rb @@ -0,0 +1,31 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Name of the LLM integration provider. + class LLMObsCustomEvalConfigIntegrationProvider + include BaseEnumModel + + OPENAI = "openai".freeze + AMAZON_BEDROCK = "amazon-bedrock".freeze + ANTHROPIC = "anthropic".freeze + AZURE_OPENAI = "azure-openai".freeze + VERTEX_AI = "vertex-ai".freeze + LLM_PROXY = "llm-proxy".freeze + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_judge_config.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_judge_config.rb new file mode 100644 index 000000000000..3db3100c6789 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_judge_config.rb @@ -0,0 +1,195 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # LLM judge configuration for a custom evaluator. + class LLMObsCustomEvalConfigLLMJudgeConfig + include BaseGenericModel + + # Criteria used to assess the pass/fail result of a custom evaluator. + attr_accessor :assessment_criteria + + # LLM inference parameters for a custom evaluator. + attr_reader :inference_params + + # Name of the last library prompt template used. + attr_accessor :last_used_library_prompt_template_name + + # Whether the library prompt template was modified. + attr_accessor :modified_library_prompt_template + + # JSON schema describing the expected output format of the LLM judge. + attr_accessor :output_schema + + # Output parsing type for a custom LLM judge evaluator. + attr_accessor :parsing_type + + # List of messages forming the LLM judge prompt template. + attr_accessor :prompt_template + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'assessment_criteria' => :'assessment_criteria', + :'inference_params' => :'inference_params', + :'last_used_library_prompt_template_name' => :'last_used_library_prompt_template_name', + :'modified_library_prompt_template' => :'modified_library_prompt_template', + :'output_schema' => :'output_schema', + :'parsing_type' => :'parsing_type', + :'prompt_template' => :'prompt_template' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'assessment_criteria' => :'LLMObsCustomEvalConfigAssessmentCriteria', + :'inference_params' => :'LLMObsCustomEvalConfigInferenceParams', + :'last_used_library_prompt_template_name' => :'String', + :'modified_library_prompt_template' => :'Boolean', + :'output_schema' => :'Hash', + :'parsing_type' => :'LLMObsCustomEvalConfigParsingType', + :'prompt_template' => :'Array' + } + end + + # List of attributes with nullable: true + # @!visibility private + def self.openapi_nullable + Set.new([ + :'last_used_library_prompt_template_name', + :'modified_library_prompt_template', + :'output_schema', + ]) + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigLLMJudgeConfig` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'assessment_criteria') + self.assessment_criteria = attributes[:'assessment_criteria'] + end + + if attributes.key?(:'inference_params') + self.inference_params = attributes[:'inference_params'] + end + + if attributes.key?(:'last_used_library_prompt_template_name') + self.last_used_library_prompt_template_name = attributes[:'last_used_library_prompt_template_name'] + end + + if attributes.key?(:'modified_library_prompt_template') + self.modified_library_prompt_template = attributes[:'modified_library_prompt_template'] + end + + if attributes.key?(:'output_schema') + self.output_schema = attributes[:'output_schema'] + end + + if attributes.key?(:'parsing_type') + self.parsing_type = attributes[:'parsing_type'] + end + + if attributes.key?(:'prompt_template') + if (value = attributes[:'prompt_template']).is_a?(Array) + self.prompt_template = value + end + end + end + + # Check to see if the all the properties in the model are valid + # @return true if the model is valid + # @!visibility private + def valid? + return false if @inference_params.nil? + true + end + + # Custom attribute writer method with validation + # @param inference_params [Object] Object to be assigned + # @!visibility private + def inference_params=(inference_params) + if inference_params.nil? + fail ArgumentError, 'invalid value for "inference_params", inference_params cannot be nil.' + end + @inference_params = inference_params + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + assessment_criteria == o.assessment_criteria && + inference_params == o.inference_params && + last_used_library_prompt_template_name == o.last_used_library_prompt_template_name && + modified_library_prompt_template == o.modified_library_prompt_template && + output_schema == o.output_schema && + parsing_type == o.parsing_type && + prompt_template == o.prompt_template && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [assessment_criteria, inference_params, last_used_library_prompt_template_name, modified_library_prompt_template, output_schema, parsing_type, prompt_template, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_provider.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_provider.rb new file mode 100644 index 000000000000..02db64c79e0c --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_provider.rb @@ -0,0 +1,145 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # LLM provider configuration for a custom evaluator. + class LLMObsCustomEvalConfigLLMProvider + include BaseGenericModel + + # AWS Bedrock-specific options for LLM provider configuration. + attr_accessor :bedrock + + # Integration account identifier. + attr_accessor :integration_account_id + + # Name of the LLM integration provider. + attr_accessor :integration_provider + + # Name of the LLM model. + attr_accessor :model_name + + # Google Vertex AI-specific options for LLM provider configuration. + attr_accessor :vertex_ai + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'bedrock' => :'bedrock', + :'integration_account_id' => :'integration_account_id', + :'integration_provider' => :'integration_provider', + :'model_name' => :'model_name', + :'vertex_ai' => :'vertex_ai' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'bedrock' => :'LLMObsCustomEvalConfigBedrockOptions', + :'integration_account_id' => :'String', + :'integration_provider' => :'LLMObsCustomEvalConfigIntegrationProvider', + :'model_name' => :'String', + :'vertex_ai' => :'LLMObsCustomEvalConfigVertexAIOptions' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigLLMProvider` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'bedrock') + self.bedrock = attributes[:'bedrock'] + end + + if attributes.key?(:'integration_account_id') + self.integration_account_id = attributes[:'integration_account_id'] + end + + if attributes.key?(:'integration_provider') + self.integration_provider = attributes[:'integration_provider'] + end + + if attributes.key?(:'model_name') + self.model_name = attributes[:'model_name'] + end + + if attributes.key?(:'vertex_ai') + self.vertex_ai = attributes[:'vertex_ai'] + end + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + bedrock == o.bedrock && + integration_account_id == o.integration_account_id && + integration_provider == o.integration_provider && + model_name == o.model_name && + vertex_ai == o.vertex_ai && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [bedrock, integration_account_id, integration_provider, model_name, vertex_ai, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_parsing_type.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_parsing_type.rb new file mode 100644 index 000000000000..fd202fb5ffd4 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_parsing_type.rb @@ -0,0 +1,27 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Output parsing type for a custom LLM judge evaluator. + class LLMObsCustomEvalConfigParsingType + include BaseEnumModel + + STRUCTURED_OUTPUT = "structured_output".freeze + JSON = "json".freeze + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_prompt_content.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_prompt_content.rb new file mode 100644 index 000000000000..614741288e62 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_prompt_content.rb @@ -0,0 +1,144 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # A content block within a prompt message. + class LLMObsCustomEvalConfigPromptContent + include BaseGenericModel + + # Content block type. + attr_reader :type + + # Value of a prompt message content block. + attr_reader :value + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'type' => :'type', + :'value' => :'value' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'type' => :'String', + :'value' => :'LLMObsCustomEvalConfigPromptContentValue' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigPromptContent` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'type') + self.type = attributes[:'type'] + end + + if attributes.key?(:'value') + self.value = attributes[:'value'] + end + end + + # Check to see if the all the properties in the model are valid + # @return true if the model is valid + # @!visibility private + def valid? + return false if @type.nil? + return false if @value.nil? + true + end + + # Custom attribute writer method with validation + # @param type [Object] Object to be assigned + # @!visibility private + def type=(type) + if type.nil? + fail ArgumentError, 'invalid value for "type", type cannot be nil.' + end + @type = type + end + + # Custom attribute writer method with validation + # @param value [Object] Object to be assigned + # @!visibility private + def value=(value) + if value.nil? + fail ArgumentError, 'invalid value for "value", value cannot be nil.' + end + @value = value + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + type == o.type && + value == o.value && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [type, value, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_prompt_content_value.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_prompt_content_value.rb new file mode 100644 index 000000000000..334e351feed9 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_prompt_content_value.rb @@ -0,0 +1,125 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Value of a prompt message content block. + class LLMObsCustomEvalConfigPromptContentValue + include BaseGenericModel + + # Text content of the message block. + attr_accessor :text + + # A tool call within a prompt message. + attr_accessor :tool_call + + # A tool call result within a prompt message. + attr_accessor :tool_call_result + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'text' => :'text', + :'tool_call' => :'tool_call', + :'tool_call_result' => :'tool_call_result' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'text' => :'String', + :'tool_call' => :'LLMObsCustomEvalConfigPromptToolCall', + :'tool_call_result' => :'LLMObsCustomEvalConfigPromptToolResult' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigPromptContentValue` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'text') + self.text = attributes[:'text'] + end + + if attributes.key?(:'tool_call') + self.tool_call = attributes[:'tool_call'] + end + + if attributes.key?(:'tool_call_result') + self.tool_call_result = attributes[:'tool_call_result'] + end + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + text == o.text && + tool_call == o.tool_call && + tool_call_result == o.tool_call_result && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [text, tool_call, tool_call_result, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_prompt_message.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_prompt_message.rb new file mode 100644 index 000000000000..4d30a6060f43 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_prompt_message.rb @@ -0,0 +1,145 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # A message in the prompt template for a custom LLM judge evaluator. + class LLMObsCustomEvalConfigPromptMessage + include BaseGenericModel + + # Text content of the message. + attr_accessor :content + + # Multi-part content blocks for the message. + attr_accessor :contents + + # Role of the message author. + attr_reader :role + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'content' => :'content', + :'contents' => :'contents', + :'role' => :'role' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'content' => :'String', + :'contents' => :'Array', + :'role' => :'String' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigPromptMessage` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'content') + self.content = attributes[:'content'] + end + + if attributes.key?(:'contents') + if (value = attributes[:'contents']).is_a?(Array) + self.contents = value + end + end + + if attributes.key?(:'role') + self.role = attributes[:'role'] + end + end + + # Check to see if the all the properties in the model are valid + # @return true if the model is valid + # @!visibility private + def valid? + return false if @role.nil? + true + end + + # Custom attribute writer method with validation + # @param role [Object] Object to be assigned + # @!visibility private + def role=(role) + if role.nil? + fail ArgumentError, 'invalid value for "role", role cannot be nil.' + end + @role = role + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + content == o.content && + contents == o.contents && + role == o.role && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [content, contents, role, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_prompt_tool_call.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_prompt_tool_call.rb new file mode 100644 index 000000000000..979a658bcb5a --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_prompt_tool_call.rb @@ -0,0 +1,135 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # A tool call within a prompt message. + class LLMObsCustomEvalConfigPromptToolCall + include BaseGenericModel + + # JSON-encoded arguments for the tool call. + attr_accessor :arguments + + # Unique identifier of the tool call. + attr_accessor :id + + # Name of the tool being called. + attr_accessor :name + + # Type of the tool call. + attr_accessor :type + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'arguments' => :'arguments', + :'id' => :'id', + :'name' => :'name', + :'type' => :'type' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'arguments' => :'String', + :'id' => :'String', + :'name' => :'String', + :'type' => :'String' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigPromptToolCall` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'arguments') + self.arguments = attributes[:'arguments'] + end + + if attributes.key?(:'id') + self.id = attributes[:'id'] + end + + if attributes.key?(:'name') + self.name = attributes[:'name'] + end + + if attributes.key?(:'type') + self.type = attributes[:'type'] + end + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + arguments == o.arguments && + id == o.id && + name == o.name && + type == o.type && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [arguments, id, name, type, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_prompt_tool_result.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_prompt_tool_result.rb new file mode 100644 index 000000000000..2c47c95a7492 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_prompt_tool_result.rb @@ -0,0 +1,135 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # A tool call result within a prompt message. + class LLMObsCustomEvalConfigPromptToolResult + include BaseGenericModel + + # Name of the tool that produced this result. + attr_accessor :name + + # The result returned by the tool. + attr_accessor :result + + # Identifier of the tool call this result corresponds to. + attr_accessor :tool_id + + # Type of the tool result. + attr_accessor :type + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'name' => :'name', + :'result' => :'result', + :'tool_id' => :'tool_id', + :'type' => :'type' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'name' => :'String', + :'result' => :'String', + :'tool_id' => :'String', + :'type' => :'String' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigPromptToolResult` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'name') + self.name = attributes[:'name'] + end + + if attributes.key?(:'result') + self.result = attributes[:'result'] + end + + if attributes.key?(:'tool_id') + self.tool_id = attributes[:'tool_id'] + end + + if attributes.key?(:'type') + self.type = attributes[:'type'] + end + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + name == o.name && + result == o.result && + tool_id == o.tool_id && + type == o.type && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [name, result, tool_id, type, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_response.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_response.rb new file mode 100644 index 000000000000..87e8104287b8 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_response.rb @@ -0,0 +1,123 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Response containing a custom LLM Observability evaluator configuration. + class LLMObsCustomEvalConfigResponse + include BaseGenericModel + + # Data object for a custom LLM Observability evaluator configuration. + attr_reader :data + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'data' => :'data' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'data' => :'LLMObsCustomEvalConfigData' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigResponse` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'data') + self.data = attributes[:'data'] + end + end + + # Check to see if the all the properties in the model are valid + # @return true if the model is valid + # @!visibility private + def valid? + return false if @data.nil? + true + end + + # Custom attribute writer method with validation + # @param data [Object] Object to be assigned + # @!visibility private + def data=(data) + if data.nil? + fail ArgumentError, 'invalid value for "data", data cannot be nil.' + end + @data = data + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + data == o.data && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [data, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_target.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_target.rb new file mode 100644 index 000000000000..ca61ee2c392b --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_target.rb @@ -0,0 +1,194 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Target application configuration for a custom evaluator. + class LLMObsCustomEvalConfigTarget + include BaseGenericModel + + # Name of the ML application this evaluator targets. + attr_reader :application_name + + # Whether the evaluator is active for the target application. + attr_reader :enabled + + # Scope at which to evaluate spans. + attr_accessor :eval_scope + + # Filter expression to select which spans to evaluate. + attr_accessor :filter + + # When true, only root spans are evaluated. + attr_accessor :root_spans_only + + # Percentage of traces to evaluate. Must be greater than 0 and at most 100. + attr_accessor :sampling_percentage + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'application_name' => :'application_name', + :'enabled' => :'enabled', + :'eval_scope' => :'eval_scope', + :'filter' => :'filter', + :'root_spans_only' => :'root_spans_only', + :'sampling_percentage' => :'sampling_percentage' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'application_name' => :'String', + :'enabled' => :'Boolean', + :'eval_scope' => :'LLMObsCustomEvalConfigEvalScope', + :'filter' => :'String', + :'root_spans_only' => :'Boolean', + :'sampling_percentage' => :'Float' + } + end + + # List of attributes with nullable: true + # @!visibility private + def self.openapi_nullable + Set.new([ + :'filter', + :'root_spans_only', + :'sampling_percentage', + ]) + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigTarget` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'application_name') + self.application_name = attributes[:'application_name'] + end + + if attributes.key?(:'enabled') + self.enabled = attributes[:'enabled'] + end + + if attributes.key?(:'eval_scope') + self.eval_scope = attributes[:'eval_scope'] + end + + if attributes.key?(:'filter') + self.filter = attributes[:'filter'] + end + + if attributes.key?(:'root_spans_only') + self.root_spans_only = attributes[:'root_spans_only'] + end + + if attributes.key?(:'sampling_percentage') + self.sampling_percentage = attributes[:'sampling_percentage'] + end + end + + # Check to see if the all the properties in the model are valid + # @return true if the model is valid + # @!visibility private + def valid? + return false if @application_name.nil? + return false if @enabled.nil? + true + end + + # Custom attribute writer method with validation + # @param application_name [Object] Object to be assigned + # @!visibility private + def application_name=(application_name) + if application_name.nil? + fail ArgumentError, 'invalid value for "application_name", application_name cannot be nil.' + end + @application_name = application_name + end + + # Custom attribute writer method with validation + # @param enabled [Object] Object to be assigned + # @!visibility private + def enabled=(enabled) + if enabled.nil? + fail ArgumentError, 'invalid value for "enabled", enabled cannot be nil.' + end + @enabled = enabled + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + application_name == o.application_name && + enabled == o.enabled && + eval_scope == o.eval_scope && + filter == o.filter && + root_spans_only == o.root_spans_only && + sampling_percentage == o.sampling_percentage && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [application_name, enabled, eval_scope, filter, root_spans_only, sampling_percentage, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_type.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_type.rb new file mode 100644 index 000000000000..435fb11b8f49 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_type.rb @@ -0,0 +1,26 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Type of the custom LLM Observability evaluator configuration resource. + class LLMObsCustomEvalConfigType + include BaseEnumModel + + EVALUATOR_CONFIG = "evaluator_config".freeze + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_update_attributes.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_update_attributes.rb new file mode 100644 index 000000000000..7dc2606aad65 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_update_attributes.rb @@ -0,0 +1,163 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Attributes for creating or updating a custom LLM Observability evaluator configuration. + class LLMObsCustomEvalConfigUpdateAttributes + include BaseGenericModel + + # Category of the evaluator. + attr_accessor :category + + # Name of the custom evaluator. If provided, must match the eval_name path parameter. + attr_accessor :eval_name + + # LLM judge configuration for a custom evaluator. + attr_accessor :llm_judge_config + + # LLM provider configuration for a custom evaluator. + attr_accessor :llm_provider + + # Target application configuration for a custom evaluator. + attr_reader :target + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'category' => :'category', + :'eval_name' => :'eval_name', + :'llm_judge_config' => :'llm_judge_config', + :'llm_provider' => :'llm_provider', + :'target' => :'target' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'category' => :'String', + :'eval_name' => :'String', + :'llm_judge_config' => :'LLMObsCustomEvalConfigLLMJudgeConfig', + :'llm_provider' => :'LLMObsCustomEvalConfigLLMProvider', + :'target' => :'LLMObsCustomEvalConfigTarget' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigUpdateAttributes` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'category') + self.category = attributes[:'category'] + end + + if attributes.key?(:'eval_name') + self.eval_name = attributes[:'eval_name'] + end + + if attributes.key?(:'llm_judge_config') + self.llm_judge_config = attributes[:'llm_judge_config'] + end + + if attributes.key?(:'llm_provider') + self.llm_provider = attributes[:'llm_provider'] + end + + if attributes.key?(:'target') + self.target = attributes[:'target'] + end + end + + # Check to see if the all the properties in the model are valid + # @return true if the model is valid + # @!visibility private + def valid? + return false if @target.nil? + true + end + + # Custom attribute writer method with validation + # @param target [Object] Object to be assigned + # @!visibility private + def target=(target) + if target.nil? + fail ArgumentError, 'invalid value for "target", target cannot be nil.' + end + @target = target + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + category == o.category && + eval_name == o.eval_name && + llm_judge_config == o.llm_judge_config && + llm_provider == o.llm_provider && + target == o.target && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [category, eval_name, llm_judge_config, llm_provider, target, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_update_data.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_update_data.rb new file mode 100644 index 000000000000..166062962154 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_update_data.rb @@ -0,0 +1,154 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Data object for creating or updating a custom LLM Observability evaluator configuration. + class LLMObsCustomEvalConfigUpdateData + include BaseGenericModel + + # Attributes for creating or updating a custom LLM Observability evaluator configuration. + attr_reader :attributes + + # Name of the evaluator. If provided, must match the eval_name path parameter. + attr_accessor :id + + # Type of the custom LLM Observability evaluator configuration resource. + attr_reader :type + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'attributes' => :'attributes', + :'id' => :'id', + :'type' => :'type' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'attributes' => :'LLMObsCustomEvalConfigUpdateAttributes', + :'id' => :'String', + :'type' => :'LLMObsCustomEvalConfigType' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigUpdateData` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'attributes') + self.attributes = attributes[:'attributes'] + end + + if attributes.key?(:'id') + self.id = attributes[:'id'] + end + + if attributes.key?(:'type') + self.type = attributes[:'type'] + end + end + + # Check to see if the all the properties in the model are valid + # @return true if the model is valid + # @!visibility private + def valid? + return false if @attributes.nil? + return false if @type.nil? + true + end + + # Custom attribute writer method with validation + # @param attributes [Object] Object to be assigned + # @!visibility private + def attributes=(attributes) + if attributes.nil? + fail ArgumentError, 'invalid value for "attributes", attributes cannot be nil.' + end + @attributes = attributes + end + + # Custom attribute writer method with validation + # @param type [Object] Object to be assigned + # @!visibility private + def type=(type) + if type.nil? + fail ArgumentError, 'invalid value for "type", type cannot be nil.' + end + @type = type + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + attributes == o.attributes && + id == o.id && + type == o.type && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [attributes, id, type, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_update_request.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_update_request.rb new file mode 100644 index 000000000000..9c3019d84fc7 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_update_request.rb @@ -0,0 +1,123 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Request to create or update a custom LLM Observability evaluator configuration. + class LLMObsCustomEvalConfigUpdateRequest + include BaseGenericModel + + # Data object for creating or updating a custom LLM Observability evaluator configuration. + attr_reader :data + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'data' => :'data' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'data' => :'LLMObsCustomEvalConfigUpdateData' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigUpdateRequest` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'data') + self.data = attributes[:'data'] + end + end + + # Check to see if the all the properties in the model are valid + # @return true if the model is valid + # @!visibility private + def valid? + return false if @data.nil? + true + end + + # Custom attribute writer method with validation + # @param data [Object] Object to be assigned + # @!visibility private + def data=(data) + if data.nil? + fail ArgumentError, 'invalid value for "data", data cannot be nil.' + end + @data = data + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + data == o.data && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [data, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_user.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_user.rb new file mode 100644 index 000000000000..59b7f5fa9654 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_user.rb @@ -0,0 +1,105 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # A Datadog user associated with a custom evaluator configuration. + class LLMObsCustomEvalConfigUser + include BaseGenericModel + + # Email address of the user. + attr_accessor :email + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'email' => :'email' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'email' => :'String' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigUser` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'email') + self.email = attributes[:'email'] + end + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + email == o.email && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [email, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_vertex_ai_options.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_vertex_ai_options.rb new file mode 100644 index 000000000000..1b0a616670fb --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_vertex_ai_options.rb @@ -0,0 +1,115 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Google Vertex AI-specific options for LLM provider configuration. + class LLMObsCustomEvalConfigVertexAIOptions + include BaseGenericModel + + # Google Cloud region. + attr_accessor :location + + # Google Cloud project ID. + attr_accessor :project + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'location' => :'location', + :'project' => :'project' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'location' => :'String', + :'project' => :'String' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigVertexAIOptions` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'location') + self.location = attributes[:'location'] + end + + if attributes.key?(:'project') + self.project = attributes[:'project'] + end + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + location == o.location && + project == o.project && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [location, project, additional_properties].hash + end + end +end