feat(ai-agents): make API keys optional for OpenAI, Anthropic, Google, and OpenAI-compatible providers

Make API keys optional in AI agent proto definitions to support deployments
using AI Gateway for authentication. When using an AI Gateway, the gateway
handles authentication, making API keys unnecessary for agent configuration.

Changes:
- Remove REQUIRED field behavior from api_key fields in Provider messages
- Add IGNORE_IF_ZERO_VALUE validation to allow empty API keys
- Update frontend forms to conditionally require API key based on gateway usage
- Add gateway configuration support in agent creation and details pages
- Regenerate proto files and OpenAPI schemas

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: alenkacz

backend/pkg/protogen/redpanda/api/dataplane/v1alpha3/ai_agent.pb.go

@@ -35,6 +35,7 @@ import { TagsFieldList } from 'components/ui/tag/tags-field-list';
 import { Loader2 } from 'lucide-react';
 import { Scope } from 'protogen/redpanda/api/dataplane/v1/secret_pb';
 import {
+  AIAgent_GatewayConfigSchema,
   AIAgent_MCPServerSchema,
   type AIAgent_Provider,
   AIAgent_Provider_AnthropicSchema,
@@ -47,9 +48,11 @@ import {
   AIAgentCreateSchema,
   CreateAIAgentRequestSchema,
 } from 'protogen/redpanda/api/dataplane/v1alpha3/ai_agent_pb';
+import { type AIGateway, AIGateway_State } from 'protogen/redpanda/api/dataplane/v1alpha3/ai_gateway_pb';
 import { useEffect, useMemo, useRef, useState } from 'react';
 import { Controller, useFieldArray, useForm } from 'react-hook-form';
 import { useCreateAIAgentMutation } from 'react-query/api/ai-agent';
+import { useListAIGatewaysQuery } from 'react-query/api/ai-gateway';
 import { useListMCPServersQuery } from 'react-query/api/remote-mcp';
 import { useCreateSecretMutation, useListSecretsQuery } from 'react-query/api/secret';
 import { toast } from 'sonner';
@@ -71,6 +74,32 @@ export const AIAgentCreatePage = () => {
     skipInvalidation: true,
   });
 
+  // Gateway detection
+  const { data: gatewaysData, isLoading: isLoadingGateways } = useListAIGatewaysQuery(
+    { pageSize: -1 },
+    { enabled: true }
+  );
+
+  const hasGatewayDeployed = useMemo(() => {
+    if (isLoadingGateways) {
+      return false;
+    }
+    return Boolean(gatewaysData?.aiGateways && gatewaysData.aiGateways.length > 0);
+  }, [gatewaysData, isLoadingGateways]);
+
+  const availableGateways = useMemo(() => {
+    if (!gatewaysData?.aiGateways) {
+      return [];
+    }
+    return gatewaysData.aiGateways
+      .filter((gw: AIGateway) => gw.state === AIGateway_State.RUNNING)
+      .map((gw: AIGateway) => ({
+        id: gw.id,
+        displayName: gw.displayName,
+        description: gw.description,
+      }));
+  }, [gatewaysData]);
+
   // Ref to ServiceAccountSelector to call createServiceAccount
   const serviceAccountSelectorRef = useRef<ServiceAccountSelectorRef>(null);
 
@@ -342,6 +371,13 @@ export const AIAgentCreatePage = () => {
         });
     }
 
+    const gatewayConfig =
+      values.gatewayId && values.gatewayId.trim() !== ''
+        ? create(AIAgent_GatewayConfigSchema, {
+            virtualGatewayId: values.gatewayId,
+          })
+        : undefined;
+
     await createAgent(
       create(CreateAIAgentRequestSchema, {
         aiAgent: create(AIAgentCreateSchema, {
@@ -359,6 +395,7 @@ export const AIAgentCreatePage = () => {
             memoryShares: selectedTier?.memory || '800M',
           },
           serviceAccount: serviceAccountConfig,
+          gateway: gatewayConfig,
         }),
       }),
       {
@@ -455,15 +492,18 @@ export const AIAgentCreatePage = () => {
                 </CardHeader>
                 <CardContent>
                   <LLMConfigSection
+                    availableGateways={availableGateways}
                     availableSecrets={availableSecrets}
                     fieldNames={{
                       provider: 'provider',
                       model: 'model',
                       apiKeySecret: 'apiKeySecret',
                       baseUrl: 'baseUrl',
                       maxIterations: 'maxIterations',
+                      gatewayId: 'gatewayId',
                     }}
                     form={form}
+                    hasGatewayDeployed={hasGatewayDeployed}
                     mode="create"
                     scopes={[Scope.MCP_SERVER, Scope.AI_AGENT]}
                     showBaseUrl={form.watch('provider') === 'openaiCompatible'}

frontend/src/components/pages/agents/create/ai-agent-create-page.tsx

@@ -50,7 +50,7 @@ export const FormSchema = z
       ),
     triggerType: z.enum(['http', 'slack', 'kafka']).default('http'),
     provider: z.enum(['openai', 'anthropic', 'google', 'openaiCompatible']).default('openai'),
-    apiKeySecret: z.string().min(1, 'API Token is required'),
+    apiKeySecret: z.string(),
     model: z.string().min(1, 'Model is required'),
     baseUrl: z.string().url('Must be a valid URL').optional().or(z.literal('')),
     maxIterations: z
@@ -77,7 +77,14 @@ export const FormSchema = z
         },
         { message: 'Subagent names must be unique' }
       ),
+    gatewayId: z
+      .string()
+      .length(20, 'Gateway ID must be exactly 20 characters')
+      .regex(/^[a-z0-9]+$/, 'Gateway ID must contain only lowercase letters and numbers')
+      .optional()
+      .or(z.literal('')),
   })
+  // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Complex validation logic with multiple conditional checks
   .superRefine((data, ctx) => {
     if (data.provider === 'openaiCompatible') {
       if (!data.baseUrl || data.baseUrl.trim() === '') {
@@ -105,6 +112,15 @@ export const FormSchema = z
         }
       }
     }
+
+    // API Key is required when not using a gateway
+    if ((!data.gatewayId || data.gatewayId.trim() === '') && (!data.apiKeySecret || data.apiKeySecret.trim() === '')) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: 'API Token is required when not using a gateway',
+        path: ['apiKeySecret'],
+      });
+    }
   });
 
 export type FormValues = z.infer<typeof FormSchema>;
@@ -126,4 +142,5 @@ export const initialValues: FormValues = {
   systemPrompt: '',
   serviceAccountName: '',
   subagents: [],
+  gatewayId: '',
 };

frontend/src/components/pages/agents/create/schemas.ts

@@ -48,6 +48,7 @@ import { ServiceAccountSection } from 'components/ui/service-account/service-acc
 import { Edit, Plus, Save, Settings, ShieldCheck, Trash2 } from 'lucide-react';
 import { Scope } from 'protogen/redpanda/api/dataplane/v1/secret_pb';
 import {
+  AIAgent_GatewayConfigSchema,
   AIAgent_MCPServerSchema,
   type AIAgent_Provider,
   AIAgent_Provider_AnthropicSchema,
@@ -59,8 +60,10 @@ import {
   AIAgentUpdateSchema,
   UpdateAIAgentRequestSchema,
 } from 'protogen/redpanda/api/dataplane/v1alpha3/ai_agent_pb';
+import { type AIGateway, AIGateway_State } from 'protogen/redpanda/api/dataplane/v1alpha3/ai_gateway_pb';
 import { useCallback, useMemo, useState } from 'react';
 import { useGetAIAgentQuery, useUpdateAIAgentMutation } from 'react-query/api/ai-agent';
+import { useListAIGatewaysQuery } from 'react-query/api/ai-gateway';
 import { type MCPServer, useListMCPServersQuery } from 'react-query/api/remote-mcp';
 import { useListSecretsQuery } from 'react-query/api/secret';
 import { toast } from 'sonner';
@@ -88,6 +91,7 @@ type LocalAIAgent = {
     systemPrompt: string;
     selectedMcpServers: string[];
   }>;
+  gatewayId?: string;
 };
 
 /**
@@ -272,6 +276,32 @@ export const AIAgentConfigurationTab = () => {
   const { data: mcpServersData } = useListMCPServersQuery();
   const { data: secretsData } = useListSecretsQuery();
 
+  // Gateway detection
+  const { data: gatewaysData, isLoading: isLoadingGateways } = useListAIGatewaysQuery(
+    { pageSize: -1 },
+    { enabled: true }
+  );
+
+  const hasGatewayDeployed = useMemo(() => {
+    if (isLoadingGateways) {
+      return false;
+    }
+    return Boolean(gatewaysData?.aiGateways && gatewaysData.aiGateways.length > 0);
+  }, [gatewaysData, isLoadingGateways]);
+
+  const availableGateways = useMemo(() => {
+    if (!gatewaysData?.aiGateways) {
+      return [];
+    }
+    return gatewaysData.aiGateways
+      .filter((gw: AIGateway) => gw.state === AIGateway_State.RUNNING)
+      .map((gw: AIGateway) => ({
+        id: gw.id,
+        displayName: gw.displayName,
+        description: gw.description,
+      }));
+  }, [gatewaysData]);
+
   const [isEditing, setIsEditing] = useState(false);
   const [editedAgentData, setEditedAgentData] = useState<LocalAIAgent | null>(null);
   const [expandedSubagent, setExpandedSubagent] = useState<string | undefined>(undefined);
@@ -341,6 +371,7 @@ export const AIAgentConfigurationTab = () => {
           systemPrompt: subagent.systemPrompt,
           selectedMcpServers: Object.values(subagent.mcpServers || {}).map((server) => server.id),
         })),
+        gatewayId: aiAgentData.aiAgent.gateway?.virtualGatewayId,
       };
     }
 
@@ -601,6 +632,13 @@ export const AIAgentConfigurationTab = () => {
       const apiKeyRef = `\${secrets.${currentData.apiKeySecret}}`;
       const updatedProvider = createUpdatedProvider(currentData.provider.provider.case, apiKeyRef, currentData.baseUrl);
 
+      const gatewayConfig =
+        currentData.gatewayId && currentData.gatewayId.trim() !== ''
+          ? create(AIAgent_GatewayConfigSchema, {
+              virtualGatewayId: currentData.gatewayId,
+            })
+          : undefined;
+
       await updateAIAgent(
         create(UpdateAIAgentRequestSchema, {
           id,
@@ -619,6 +657,7 @@ export const AIAgentConfigurationTab = () => {
             mcpServers: mcpServersMap,
             subagents: subagentsMap,
             tags: tagsMap,
+            gateway: gatewayConfig,
           }),
           updateMask: create(FieldMaskSchema, {
             paths: [
@@ -630,6 +669,7 @@ export const AIAgentConfigurationTab = () => {
               'system_prompt',
               'service_account',
               'resources',
+              'gateway',
               'mcp_servers',
               'subagents',
               'tags',
@@ -1007,10 +1047,35 @@ export const AIAgentConfigurationTab = () => {
             <CardContent className="px-4 pb-4">
               {isEditing ? (
                 <div className="space-y-4">
+                  {/* Gateway Selection - only show if gateways deployed */}
+                  {hasGatewayDeployed && availableGateways.length > 0 && (
+                    <div className="space-y-2">
+                      <Label>AI Gateway</Label>
+                      <Text className="text-muted-foreground text-xs">Route requests through an AI Gateway</Text>
+                      <Select
+                        onValueChange={(value) => updateField({ gatewayId: value || undefined })}
+                        value={displayData.gatewayId || ''}
+                      >
+                        <SelectTrigger>
+                          <SelectValue placeholder="No gateway" />
+                        </SelectTrigger>
+                        <SelectContent>
+                          <SelectItem value="">No gateway</SelectItem>
+                          {availableGateways.map((gw: { id: string; displayName: string; description: string }) => (
+                            <SelectItem key={gw.id} value={gw.id}>
+                              {gw.displayName}
+                            </SelectItem>
+                          ))}
+                        </SelectContent>
+                      </Select>
+                    </div>
+                  )}
+
                   {/* Provider - now editable */}
                   <div className="space-y-2">
                     <Label htmlFor="provider">Provider</Label>
                     <Select
+                      disabled={!!displayData.gatewayId}
                       onValueChange={(value: 'openai' | 'anthropic' | 'google' | 'openaiCompatible') => {
                         const newProviderData = MODEL_OPTIONS_BY_PROVIDER[value];
                         const firstModel =
@@ -1071,12 +1136,17 @@ export const AIAgentConfigurationTab = () => {
                     <Label htmlFor="model">Model</Label>
                     {displayData.provider?.provider.case === 'openaiCompatible' ? (
                       <Input
+                        disabled={!!displayData.gatewayId}
                         onChange={(e) => updateField({ model: e.target.value })}
                         placeholder="Enter model name (e.g., llama-3.1-70b)"
                         value={displayData.model}
                       />
                     ) : (
-                      <Select onValueChange={(value) => updateField({ model: value })} value={displayData.model}>
+                      <Select
+                        disabled={!!displayData.gatewayId}
+                        onValueChange={(value) => updateField({ model: value })}
+                        value={displayData.model}
+                      >
                         <SelectTrigger>
                           <SelectValue>
                             {Boolean(displayData.model) && detectProvider(displayData.model) ? (
@@ -1137,23 +1207,25 @@ export const AIAgentConfigurationTab = () => {
                     )}
                   </div>
 
-                  {/* API Token */}
-                  <div className="space-y-2">
-                    <Label htmlFor="apiKeySecret">API Token</Label>
-                    <div className="[&>div]:flex-col [&>div]:items-stretch [&>div]:gap-2">
-                      <SecretSelector
-                        availableSecrets={availableSecrets}
-                        customText={AI_AGENT_SECRET_TEXT}
-                        onChange={(value) => updateField({ apiKeySecret: value })}
-                        placeholder="Select from secrets store or create new"
-                        scopes={[Scope.MCP_SERVER, Scope.AI_AGENT]}
-                        value={displayData.apiKeySecret}
-                      />
+                  {/* API Token - HIDE if using gateway */}
+                  {!displayData.gatewayId && (
+                    <div className="space-y-2">
+                      <Label htmlFor="apiKeySecret">API Token</Label>
+                      <div className="[&>div]:flex-col [&>div]:items-stretch [&>div]:gap-2">
+                        <SecretSelector
+                          availableSecrets={availableSecrets}
+                          customText={AI_AGENT_SECRET_TEXT}
+                          onChange={(value) => updateField({ apiKeySecret: value })}
+                          placeholder="Select from secrets store or create new"
+                          scopes={[Scope.MCP_SERVER, Scope.AI_AGENT]}
+                          value={displayData.apiKeySecret}
+                        />
+                      </div>
                     </div>
-                  </div>
+                  )}
 
-                  {/* Base URL - only show for openaiCompatible */}
-                  {displayData.provider?.provider.case === 'openaiCompatible' && (
+                  {/* Base URL - only show for openaiCompatible and when not using gateway */}
+                  {!displayData.gatewayId && displayData.provider?.provider.case === 'openaiCompatible' && (
                     <div className="space-y-2">
                       <Label htmlFor="baseUrl">Base URL (required)</Label>
                       <Input
@@ -1181,6 +1253,21 @@ export const AIAgentConfigurationTab = () => {
                 </div>
               ) : (
                 <div className="space-y-4">
+                  {/* Gateway - only show if gateways deployed */}
+                  {hasGatewayDeployed && availableGateways.length > 0 && (
+                    <div className="space-y-2">
+                      <Label>AI Gateway</Label>
+                      <div className="flex h-10 items-center rounded-md border border-gray-200 bg-gray-50 px-3 py-2">
+                        <Text variant="default">
+                          {displayData.gatewayId
+                            ? availableGateways.find((gw: { id: string; displayName: string; description: string }) => gw.id === displayData.gatewayId)?.displayName ||
+                              displayData.gatewayId
+                            : 'None'}
+                        </Text>
+                      </div>
+                    </div>
+                  )}
+
                   <div className="space-y-2">
                     <Label>Provider</Label>
                     <div className="flex h-10 items-center rounded-md border border-gray-200 bg-gray-50 px-3 py-2">
@@ -1198,20 +1285,25 @@ export const AIAgentConfigurationTab = () => {
                       <AIAgentModel model={displayData.model} />
                     </div>
                   </div>
-                  <div className="space-y-2">
-                    <Label>API Token</Label>
-                    <div className="flex h-10 items-center rounded-md border border-gray-200 bg-gray-50 px-3 py-2">
-                      <Text variant="default">{displayData.apiKeySecret || 'No secret configured'}</Text>
-                    </div>
-                  </div>
-                  {agent.provider?.provider.case === 'openaiCompatible' && displayData.baseUrl && (
+                  {/* API Token - HIDE if using gateway */}
+                  {!displayData.gatewayId && (
                     <div className="space-y-2">
-                      <Label>Base URL</Label>
+                      <Label>API Token</Label>
                       <div className="flex h-10 items-center rounded-md border border-gray-200 bg-gray-50 px-3 py-2">
-                        <Text variant="default">{displayData.baseUrl}</Text>
+                        <Text variant="default">{displayData.apiKeySecret || 'No secret configured'}</Text>
                       </div>
                     </div>
                   )}
+                  {!displayData.gatewayId &&
+                    agent.provider?.provider.case === 'openaiCompatible' &&
+                    displayData.baseUrl && (
+                      <div className="space-y-2">
+                        <Label>Base URL</Label>
+                        <div className="flex h-10 items-center rounded-md border border-gray-200 bg-gray-50 px-3 py-2">
+                          <Text variant="default">{displayData.baseUrl}</Text>
+                        </div>
+                      </div>
+                    )}
                   <div className="space-y-2">
                     <Label>Max Iterations</Label>
                     <div className="flex h-10 items-center rounded-md border border-gray-200 bg-gray-50 px-3 py-2">