diff --git a/apps/expo-example/package.json b/apps/expo-example/package.json index a09c8210..561c80b2 100644 --- a/apps/expo-example/package.json +++ b/apps/expo-example/package.json @@ -40,6 +40,7 @@ "expo-symbols": "~1.0.8", "expo-system-ui": "^6.0.9", "jotai": "^2.12.2", + "@react-native-ai/json-ui": "workspace:*", "llama.rn": "^0.10.1", "react": "19.1.0", "react-native": "0.81.5", diff --git a/apps/expo-example/src/components/adapters/appleSetupAdapter.ts b/apps/expo-example/src/components/adapters/appleSetupAdapter.ts index 499813cf..b0e3ce08 100644 --- a/apps/expo-example/src/components/adapters/appleSetupAdapter.ts +++ b/apps/expo-example/src/components/adapters/appleSetupAdapter.ts @@ -1,15 +1,13 @@ import type { LanguageModelV3, SpeechModelV3 } from '@ai-sdk/provider' +import type { Tool } from '@ai-sdk/provider-utils' import { apple, createAppleProvider } from '@react-native-ai/apple' -import { ToolSet } from 'ai' import type { SetupAdapter } from '../../config/providers.common' export const createAppleLanguageSetupAdapter = ( - tools: ToolSet = {} + tools: Record ): SetupAdapter => { - const apple = createAppleProvider({ - availableTools: tools, - }) + const apple = createAppleProvider({ availableTools: tools }) const model = apple.languageModel() return { model, diff --git a/apps/expo-example/src/components/adapters/llamaModelSetupAdapter.ts b/apps/expo-example/src/components/adapters/llamaModelSetupAdapter.ts index 6900f090..24a5dda4 100644 --- a/apps/expo-example/src/components/adapters/llamaModelSetupAdapter.ts +++ b/apps/expo-example/src/components/adapters/llamaModelSetupAdapter.ts @@ -1,6 +1,5 @@ import type { LanguageModelV3 } from '@ai-sdk/provider' import { llama } from '@react-native-ai/llama' -import { ToolSet } from 'ai' import { downloadModel, @@ -11,8 +10,7 @@ import type { Availability, SetupAdapter } from '../../config/providers.common' import { isLlamaModelDownloaded } from '../../utils/llamaStorageUtils' export const createLlamaLanguageSetupAdapter = ( - hfModelId: string, - tools: ToolSet = {} + hfModelId: string ): SetupAdapter => { const modelPath = getModelPath(hfModelId) const model = llama.languageModel(modelPath, { diff --git a/apps/expo-example/src/config/providers.common.ts b/apps/expo-example/src/config/providers.common.ts index cf861b29..690c4eae 100644 --- a/apps/expo-example/src/config/providers.common.ts +++ b/apps/expo-example/src/config/providers.common.ts @@ -3,7 +3,6 @@ import type { LanguageModelV3, SpeechModelV3 } from '@ai-sdk/provider' import { createLlamaLanguageSetupAdapter } from '../components/adapters/llamaModelSetupAdapter' import { createLlamaSpeechSetupAdapter } from '../components/adapters/llamaSpeechSetupAdapter' import { createMLCLanguageSetupAdapter } from '../components/adapters/mlcModelSetupAdapter' -import { toolDefinitions } from '../tools' export type Availability = 'yes' | 'no' | 'availableForDownload' @@ -42,8 +41,7 @@ export const commonLanguageAdapters: SetupAdapter[] = [ 'ggml-org/SmolLM3-3B-GGUF/SmolLM3-Q4_K_M.gguf' ), createLlamaLanguageSetupAdapter( - 'Qwen/Qwen2.5-3B-Instruct-GGUF/qwen2.5-3b-instruct-q3_k_m.gguf', - toolDefinitions + 'Qwen/Qwen2.5-3B-Instruct-GGUF/qwen2.5-3b-instruct-q3_k_m.gguf' ), createMLCLanguageSetupAdapter('Llama-3.2-1B-Instruct'), createMLCLanguageSetupAdapter('Llama-3.2-3B-Instruct'), diff --git a/apps/expo-example/src/screens/ChatScreen/ChatMessageBubble.tsx b/apps/expo-example/src/screens/ChatScreen/ChatMessageBubble.tsx index 96e912d0..e5075463 100644 --- a/apps/expo-example/src/screens/ChatScreen/ChatMessageBubble.tsx +++ b/apps/expo-example/src/screens/ChatScreen/ChatMessageBubble.tsx @@ -1,37 +1,104 @@ import React from 'react' -import { StyleSheet, Text, View } from 'react-native' +import { Pressable, StyleSheet, Text, View } from 'react-native' import { colors } from '../../theme/colors' type ChatMessageBubbleProps = { content: string isUser: boolean + messageType?: 'text' | 'toolExecution' + toolExecution?: { + toolName: string + payload: unknown + result?: unknown + } } -export function ChatMessageBubble({ content, isUser }: ChatMessageBubbleProps) { +export function ChatMessageBubble({ + content, + isUser, + messageType = 'text', + toolExecution, +}: ChatMessageBubbleProps) { + const [expanded, setExpanded] = React.useState(false) + const isToolHint = messageType === 'toolExecution' + const toolLabel = toolExecution + ? `Tool executed: ${toolExecution.toolName}` + : content + const payload = toolExecution?.payload + ? JSON.stringify(toolExecution.payload, null, 2) + : '' + const result = + toolExecution?.result !== undefined + ? JSON.stringify(toolExecution.result, null, 2) + : '' + const hasPayload = payload !== '{}' && payload.length > 0 + const hasResult = result.length > 0 + const hasDetails = hasPayload || hasResult + return ( - hasDetails && setExpanded((prev) => !prev)} style={[ styles.messageBubble, - isUser ? styles.messageBubbleUser : styles.messageBubbleAssistant, + isToolHint + ? styles.messageBubbleToolHint + : isUser + ? styles.messageBubbleUser + : styles.messageBubbleAssistant, ]} > - - {content} - - + {isToolHint ? ( + + + {toolLabel} + + {hasDetails && ( + {expanded ? '[-]' : '[+]'} + )} + + ) : ( + + {content} + + )} + {isToolHint && expanded && hasPayload ? ( + + args + + {payload} + + + ) : null} + {isToolHint && expanded && hasResult ? ( + + result + + {result} + + + ) : null} + ) } @@ -59,6 +126,10 @@ const styles = StyleSheet.create({ messageBubbleAssistant: { backgroundColor: colors.secondarySystemBackground as any, }, + messageBubbleToolHint: { + backgroundColor: colors.tertiarySystemFill as any, + borderRadius: 14, + }, messageText: { fontSize: 16, lineHeight: 22, @@ -69,4 +140,35 @@ const styles = StyleSheet.create({ messageTextAssistant: { color: colors.label as any, }, + messageTextToolHint: { + color: colors.secondaryLabel as any, + fontSize: 14, + lineHeight: 20, + }, + toolHintHeader: { + flexDirection: 'row', + alignItems: 'center', + justifyContent: 'space-between', + gap: 8, + }, + expandIcon: { + color: colors.tertiaryLabel as any, + fontSize: 12, + }, + payloadText: { + marginTop: 8, + fontFamily: 'Menlo', + fontSize: 12, + lineHeight: 18, + color: colors.tertiaryLabel as any, + }, + detailBlock: { + marginTop: 8, + }, + detailLabel: { + fontSize: 11, + color: colors.tertiaryLabel as any, + textTransform: 'uppercase', + letterSpacing: 0.5, + }, }) diff --git a/apps/expo-example/src/screens/ChatScreen/ChatMessages.tsx b/apps/expo-example/src/screens/ChatScreen/ChatMessages.tsx index 2d36a1c5..52cb2f58 100644 --- a/apps/expo-example/src/screens/ChatScreen/ChatMessages.tsx +++ b/apps/expo-example/src/screens/ChatScreen/ChatMessages.tsx @@ -1,3 +1,4 @@ +import { GenerativeUIView } from '@react-native-ai/json-ui' import React, { useEffect, useRef } from 'react' import { ScrollView, StyleSheet, View } from 'react-native' import { useKeyboardHandler } from 'react-native-keyboard-controller' @@ -8,6 +9,7 @@ import Reanimated, { import { useSafeAreaInsets } from 'react-native-safe-area-context' import { scheduleOnRN } from 'react-native-worklets' +import { getChatUISpecFromChats, useChatStore } from '../../store/chatStore' import { ChatEmptyState } from './ChatEmptyState' import { ChatInputBar } from './ChatInputBar' import { ChatMessageBubble } from './ChatMessageBubble' @@ -16,6 +18,12 @@ type ChatMessage = { id: string role: string content: string + type?: 'text' | 'toolExecution' + toolExecution?: { + toolName: string + payload: unknown + result?: unknown + } } type ChatMessagesProps = { @@ -62,6 +70,8 @@ export function ChatMessages({ useEffect(scrollToBottom, [messages.length]) + const { chats, currentChatId } = useChatStore() + return ( <> ) : ( @@ -81,8 +91,20 @@ export function ChatMessages({ key={message.id} content={message.content} isUser={message.role === 'user'} + messageType={message.type} + toolExecution={message.toolExecution} /> ))} + + )} diff --git a/apps/expo-example/src/screens/ChatScreen/index.tsx b/apps/expo-example/src/screens/ChatScreen/index.tsx index 9f5af853..1663ccb9 100644 --- a/apps/expo-example/src/screens/ChatScreen/index.tsx +++ b/apps/expo-example/src/screens/ChatScreen/index.tsx @@ -1,13 +1,22 @@ import { TrueSheet } from '@lodev09/react-native-true-sheet' +import { type createAppleProvider } from '@react-native-ai/apple' +import { + buildGenUISystemPrompt, + createGenUITools, +} from '@react-native-ai/json-ui' import { stepCountIs, streamText } from 'ai' -import React, { useEffect, useRef, useState } from 'react' +import React, { useCallback, useEffect, useRef, useState } from 'react' import { StyleSheet, View } from 'react-native' import { SafeAreaView } from 'react-native-safe-area-context' -import { useChatStore } from '../../store/chatStore' +import { getChatUISpecFromChats, useChatStore } from '../../store/chatStore' import { useProviderStore } from '../../store/providerStore' import { colors } from '../../theme/colors' -import { toolDefinitions } from '../../tools' +import { + setToolExecutionReporter, + toolDefinitions, + withToolProxy, +} from '../../tools' import { ChatHeader } from './ChatHeader' import { ChatMessages } from './ChatMessages' import { ModelAvailableForDownload } from './ModelAvailableForDownload' @@ -16,8 +25,21 @@ import { ModelUnavailable } from './ModelUnavailable' import { SettingsSheet } from './SettingsSheet' export default function ChatScreen() { - const { currentChat, chatSettings, addMessages, updateMessageContent } = - useChatStore() + const { + chats, + currentChat, + chatSettings, + addMessages, + addToolExecutionMessage, + updateMessageContent, + updateChatUISpec, + } = useChatStore() + const chatsRef = useRef(chats) + chatsRef.current = chats + const getSpec = useCallback( + (chatId: string) => getChatUISpecFromChats(chatsRef.current, chatId), + [] + ) const { adapters, availability } = useProviderStore() const { @@ -57,38 +79,75 @@ export default function ChatScreen() { setIsGenerating(true) try { + const genUITools = createGenUITools({ + contextId: chatId, + getSpec, + updateSpec: updateChatUISpec, + toolWrapper: withToolProxy as any, + }) + const tools = { + ...Object.fromEntries( + enabledToolIds + .filter((id) => toolDefinitions[id]) + .map((id) => [id, toolDefinitions[id]]) + ), + ...genUITools, + } + if ('updateTools' in selectedAdapter.model) { + ;( + selectedAdapter.model as ReturnType< + ReturnType['languageModel'] + > + ).updateTools(tools) + } + setToolExecutionReporter(({ toolName, args, result }) => { + addToolExecutionMessage(chatId, toolName, args, result) + }) const result = streamText({ model: selectedAdapter.model, messages: [ - ...baseMessages.map((message) => ({ - role: message.role, - content: message.content, - })), + ...baseMessages + .filter((message) => message.type !== 'toolExecution') + .map((message) => ({ + role: message.role, + content: message.content, + })), { role: 'user', content: userInput }, ], - tools: Object.fromEntries( - enabledToolIds - .filter((id) => toolDefinitions[id]) - .map((id) => [id, toolDefinitions[id]]) - ), + tools, temperature, stopWhen: stepCountIs(maxSteps), abortSignal: signal, + system: buildGenUISystemPrompt({ + additionalInstructions: + 'If the user asks, tell who you are (assistant) and what is this (Callstack AI demo app).', + }), }) let accumulated = '' for await (const chunk of result.textStream) { if (signal.aborted) break + if (!chunk) continue accumulated += chunk updateMessageContent(chatId, assistantMessageId, accumulated) } + + if (accumulated.trim().length === 0) { + updateMessageContent( + chatId, + assistantMessageId, + 'The LLM did not yield a response. Please try again.' + ) + } } catch (error) { // Don't show error if user cancelled if (signal.aborted) return const message = error instanceof Error ? error.message : 'Failed to generate response' updateMessageContent(chatId, assistantMessageId, `Error: ${message}`) + abortControllerRef.current?.abort() } finally { + setToolExecutionReporter(null) setIsGenerating(false) } } diff --git a/apps/expo-example/src/store/chatStore.ts b/apps/expo-example/src/store/chatStore.ts index 2b8356c3..8d07be3d 100644 --- a/apps/expo-example/src/store/chatStore.ts +++ b/apps/expo-example/src/store/chatStore.ts @@ -8,12 +8,19 @@ import { toolDefinitions } from '../tools' const storage = createJSONStorage(() => AsyncStorage) export type MessageRole = 'user' | 'assistant' +export type MessageType = 'text' | 'toolExecution' export type Message = { id: string role: MessageRole content: string createdAt: string + type?: MessageType + toolExecution?: { + toolName: string + payload: unknown + result?: unknown + } } export type ChatSettings = { @@ -23,12 +30,85 @@ export type ChatSettings = { enabledToolIds: string[] } +/** Single element in the generative UI tree (id is the key in elements). */ +export type ChatUIElement = { + type: string + props: Record + children?: string[] +} + +/** Generative UI spec: root id + elements map. Root element id is always "root" (undeletable View with flex: 1). */ +export type ChatUISpec = { + root: string + elements: Record +} + +export const GEN_UI_ROOT_ID = 'root' + +/** Default root-only spec so tools and view always have a root to work with. */ +export const DEFAULT_GEN_UI_SPEC: ChatUISpec = { + root: GEN_UI_ROOT_ID, + elements: { + [GEN_UI_ROOT_ID]: { + type: 'Container', + props: { flex: 1 }, + children: [], + }, + }, +} + +const cloneGenUISpec = (spec: ChatUISpec): ChatUISpec => ({ + root: spec.root, + elements: Object.fromEntries( + Object.entries(spec.elements).map(([id, element]) => [ + id, + { + ...element, + props: { ...(element.props ?? {}) }, + children: [...(element.children ?? [])], + }, + ]) + ), +}) + +/** Ensures spec has an undeletable root Container with flex: 1. */ +export function normalizeGenUISpec( + spec: ChatUISpec | null | undefined +): ChatUISpec | null { + if (!spec) return null + const elements = { ...spec.elements } + if (!elements[GEN_UI_ROOT_ID]) { + elements[GEN_UI_ROOT_ID] = { + type: 'Container', + props: { flex: 1 }, + children: [], + } + } + const rootId = spec.root || GEN_UI_ROOT_ID + if (!elements[rootId]) elements[rootId] = elements[GEN_UI_ROOT_ID] + return { root: rootId, elements } +} + +/** Get normalized UI spec for a chat by id. Returns default root spec when chat has no uiSpec. */ +export function getChatUISpecFromChats( + chats: Chat[], + chatId: string +): ChatUISpec { + const chat = chats.find((c) => c.id === chatId) + const normalized = normalizeGenUISpec(chat?.uiSpec ?? null) + return normalized + ? cloneGenUISpec(normalized) + : cloneGenUISpec(DEFAULT_GEN_UI_SPEC) +} + export type Chat = { id: string title: string messages: Message[] createdAt: string settings: ChatSettings + /** Generative UI spec (root + elements). Root node id "root" is always present. */ + uiSpec?: ChatUISpec | null } const DEFAULT_SETTINGS: ChatSettings = { @@ -60,6 +140,9 @@ export function useChatStore() { const currentChat = chats.find((chat) => chat.id === currentChatId) + const getSafeChats = (value: unknown) => + Array.isArray(value) ? value : chats + const resetPendingSettings = () => { setPendingSettings({ ...DEFAULT_SETTINGS }) } @@ -83,11 +166,11 @@ export function useChatStore() { createdAt: new Date().toISOString(), })) - const isNewChat = !currentChatId + const isNewChat = !currentChatId || !currentChat if (isNewChat) { const firstUserMessage = messages.find((m) => m.role === 'user') setChats((prev) => { - const arr = Array.isArray(prev) ? prev : [] + const arr = getSafeChats(prev) return [ { id: chatId, @@ -97,6 +180,7 @@ export function useChatStore() { messages: newMessages, createdAt: new Date().toISOString(), settings: { ...pendingSettings }, + uiSpec: undefined, }, ...arr, ] @@ -105,7 +189,7 @@ export function useChatStore() { resetPendingSettings() } else { setChats((prev) => { - const arr = Array.isArray(prev) ? prev : [] + const arr = getSafeChats(prev) return arr.map((chat) => chat.id === chatId ? { @@ -120,14 +204,58 @@ export function useChatStore() { return { chatId, messageIds: newMessages.map((m) => m.id) } } + const addToolExecutionMessage = ( + chatId: string, + toolName: string, + payload: unknown, + result?: unknown + ) => { + const toolMessage: Message = { + id: createId(), + role: 'assistant', + type: 'toolExecution', + content: `Executed tool: ${toolName}`, + createdAt: new Date().toISOString(), + toolExecution: { + toolName, + payload, + result, + }, + } + + setChats((prev) => { + return (prev as Chat[]).map((chat) => + chat.id === chatId + ? { + ...chat, + messages: [ + ...chat.messages.slice(0, -1), + toolMessage, + ...chat.messages.slice(-1), + ], + } + : chat + ) + }) + } + + const updateChatUISpec = (chatId: string, spec: ChatUISpec | null) => { + const normalized = normalizeGenUISpec(spec) + + setChats((prev) => { + return (prev as Chat[]).map((chat) => + chat.id === chatId ? { ...chat, uiSpec: normalized ?? undefined } : chat + ) + }) + } + const updateMessageContent = ( chatId: string, messageId: string, content: string ) => { setChats((prev) => { - const arr = Array.isArray(prev) ? prev : [] - return arr.map((chat) => + return (prev as Chat[]).map((chat) => chat.id === chatId ? { ...chat, @@ -146,7 +274,7 @@ export function useChatStore() { return } setChats((prev) => { - const arr = Array.isArray(prev) ? prev : [] + const arr = getSafeChats(prev) return arr.map((chat) => chat.id === currentChatId ? { ...chat, settings: { ...chat.settings, ...updates } } @@ -166,15 +294,20 @@ export function useChatStore() { }) } + const hasGenerativeUI = Boolean(currentChat?.uiSpec) + return { chats, currentChatId, currentChat, chatSettings, + hasGenerativeUI, selectChat: setCurrentChatId, deleteChat, addMessages, + addToolExecutionMessage, updateMessageContent, + updateChatUISpec, updateChatSettings, toggleTool, } diff --git a/apps/expo-example/src/store/providerStore.ts b/apps/expo-example/src/store/providerStore.ts index fb9c0b67..695058c9 100644 --- a/apps/expo-example/src/store/providerStore.ts +++ b/apps/expo-example/src/store/providerStore.ts @@ -5,7 +5,6 @@ import { atomWithRefresh } from 'jotai/utils' import { createLlamaLanguageSetupAdapter } from '../components/adapters/llamaModelSetupAdapter' import { languageAdapters } from '../config/providers' import { type Availability } from '../config/providers.common' -import { toolDefinitions } from '../tools' export type CustomModel = { id: string @@ -18,7 +17,7 @@ const customModelsAtom = atom([]) const adaptersAtom = atom((get) => { const customModels = get(customModelsAtom).map((model) => - createLlamaLanguageSetupAdapter(model.url, toolDefinitions) + createLlamaLanguageSetupAdapter(model.url) ) return [...languageAdapters, ...customModels] }) diff --git a/apps/expo-example/src/tools.ts b/apps/expo-example/src/tools.ts index 9439c1c9..fae07010 100644 --- a/apps/expo-example/src/tools.ts +++ b/apps/expo-example/src/tools.ts @@ -2,6 +2,63 @@ import { tool } from 'ai' import * as Calendar from 'expo-calendar' import { z } from 'zod' +type ToolExecutionReporter = (event: { + toolName: string + args: unknown + result?: unknown +}) => void + +let toolExecutionReporter: ToolExecutionReporter | null = null + +export function setToolExecutionReporter( + reporter: ToolExecutionReporter | null +) { + toolExecutionReporter = reporter +} + +/** + * Wraps a tool execute function: on throw, logs the error and returns { error: message }. + * Reports tool execution args & results to the toolExecutionReporter. + */ +export function withToolProxy( + toolName: string, + execute: (args: TArgs) => Promise +): (args: TArgs) => Promise { + return async (args: TArgs) => { + try { + console.log('[tools] Executing tool', toolName, args) + const result = await execute(args) + try { + console.log( + '[tools] Finished tool execution with success', + toolName, + args, + result + ) + toolExecutionReporter?.({ toolName, args, result }) + } catch (reportError) { + console.warn('[tools] Failed to report tool execution', reportError) + } + return result + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + console.error(`[tool ${toolName}]`, err) + try { + console.log( + '[tools] Finished tool execution with error', + toolName, + args, + { error: message } + ) + toolExecutionReporter?.({ toolName, args, result: { error: message } }) + } catch (reportError) { + console.warn('[tools] Failed to report tool execution', reportError) + } + return { error: message } + } + } +} + const createCalendarEvent = tool({ title: 'createCalendarEvent', description: 'Create a new calendar event', @@ -11,27 +68,30 @@ const createCalendarEvent = tool({ time: z.string().optional().describe('Event time (HH:MM)'), duration: z.number().optional().describe('Duration in minutes'), }), - execute: async ({ title, date, time, duration = 60 }) => { - await Calendar.requestCalendarPermissionsAsync() + execute: withToolProxy( + 'createCalendarEvent', + async ({ title, date, time, duration = 60 }) => { + await Calendar.requestCalendarPermissionsAsync() - const calendars = await Calendar.getCalendarsAsync( - Calendar.EntityTypes.EVENT - ) + const calendars = await Calendar.getCalendarsAsync( + Calendar.EntityTypes.EVENT + ) - const eventDate = new Date(date) - if (time) { - const [hours, minutes] = time.split(':').map(Number) - eventDate.setHours(hours, minutes) - } + const eventDate = new Date(date) + if (time) { + const [hours, minutes] = time.split(':').map(Number) + eventDate.setHours(hours, minutes) + } - await Calendar.createEventAsync(calendars[0].id, { - title, - startDate: eventDate, - endDate: new Date(eventDate.getTime() + duration * 60 * 1000), - }) + await Calendar.createEventAsync(calendars[0].id, { + title, + startDate: eventDate, + endDate: new Date(eventDate.getTime() + duration * 60 * 1000), + }) - return { message: `Created "${title}"` } - }, + return { message: `Created "${title}"` } + } + ), }) const checkCalendarEvents = tool({ @@ -40,7 +100,7 @@ const checkCalendarEvents = tool({ inputSchema: z.object({ days: z.number().optional().describe('Number of days to look ahead'), }), - execute: async ({ days = 7 }) => { + execute: withToolProxy('checkCalendarEvents', async ({ days = 7 }) => { await Calendar.requestCalendarPermissionsAsync() const calendars = await Calendar.getCalendarsAsync( @@ -60,16 +120,16 @@ const checkCalendarEvents = tool({ title: event.title, date: event.startDate, })) - }, + }), }) const getCurrentTime = tool({ title: 'getCurrentTime', description: 'Get current time and date', inputSchema: z.object({}), - execute: async () => { + execute: withToolProxy('getCurrentTime', async () => { return `Current time is: ${new Date().toUTCString()}` - }, + }), }) export const toolDefinitions = { diff --git a/apps/expo-example/src/ui/genUiNodes.ts b/apps/expo-example/src/ui/genUiNodes.ts new file mode 100644 index 00000000..5812bdc8 --- /dev/null +++ b/apps/expo-example/src/ui/genUiNodes.ts @@ -0,0 +1,6 @@ +export { + GEN_UI_NODE_HINTS, + GEN_UI_NODE_NAMES, + GEN_UI_NODE_NAMES_THAT_SUPPORT_CHILDREN, + GEN_UI_STYLES, +} from '@react-native-ai/json-ui' diff --git a/apps/expo-example/tsconfig.json b/apps/expo-example/tsconfig.json index 2b511d42..933518fb 100644 --- a/apps/expo-example/tsconfig.json +++ b/apps/expo-example/tsconfig.json @@ -5,7 +5,10 @@ "moduleSuffixes": [".android", ".ios", ""], "paths": { "@react-native-ai/apple": ["../../packages/apple-llm/src"], - "@react-native-ai/mlc": ["../../packages/mlc/src"] + "@react-native-ai/mlc": ["../../packages/mlc/src"], + "@react-native-ai/json-ui": [ + "../../packages/@react-native-ai/json-ui/src" + ] } }, "include": ["src/**/*", "nativewind-env.d.ts"] diff --git a/bun.lock b/bun.lock index 00c7136e..8a02077b 100644 --- a/bun.lock +++ b/bun.lock @@ -39,6 +39,7 @@ "@expo/vector-icons": "^15.0.3", "@lodev09/react-native-true-sheet": "^3.8.1", "@react-native-ai/apple": "workspace:*", + "@react-native-ai/json-ui": "workspace:*", "@react-native-ai/llama": "workspace:*", "@react-native-ai/mlc": "workspace:*", "@react-native-async-storage/async-storage": "^2.2.0", @@ -124,6 +125,18 @@ "react-native": "*", }, }, + "packages/json-ui": { + "name": "@react-native-ai/json-ui", + "version": "1.0.0-alpha.1", + "dependencies": { + "ai": "^6.0.0", + "zod": "^4.0.0", + }, + "peerDependencies": { + "react": "*", + "react-native": "*", + }, + }, "packages/llama": { "name": "@react-native-ai/llama", "version": "0.12.0", @@ -745,6 +758,8 @@ "@react-native-ai/example": ["@react-native-ai/example@workspace:apps/expo-example"], + "@react-native-ai/json-ui": ["@react-native-ai/json-ui@workspace:packages/json-ui"], + "@react-native-ai/llama": ["@react-native-ai/llama@workspace:packages/llama"], "@react-native-ai/mlc": ["@react-native-ai/mlc@workspace:packages/mlc"], diff --git a/packages/apple-llm/src/ai-sdk.ts b/packages/apple-llm/src/ai-sdk.ts index f2d96704..ff85d1c5 100644 --- a/packages/apple-llm/src/ai-sdk.ts +++ b/packages/apple-llm/src/ai-sdk.ts @@ -17,7 +17,7 @@ import { generateId, jsonSchema, parseJSON, - type Tool as ToolDefinition, + Tool as FullToolDefinition, ToolCallOptions, } from '@ai-sdk/provider-utils' @@ -28,12 +28,12 @@ import NativeAppleTranscription from './NativeAppleTranscription' import NativeAppleUtils from './NativeAppleUtils' type Tool = LanguageModelV3FunctionTool | LanguageModelV3ProviderTool -type ToolSet = Record +type ToolDefinitionSet = Record export function createAppleProvider({ availableTools, }: { - availableTools?: ToolSet + availableTools?: ToolDefinitionSet } = {}) { const createLanguageModel = () => { return new AppleLLMChatLanguageModel(availableTools) @@ -235,10 +235,10 @@ class AppleLLMChatLanguageModel implements LanguageModelV3 { readonly provider = 'apple' readonly modelId = 'system-default' - private tools: ToolSet + private tools: ToolDefinitionSet = {} - constructor(tools: ToolSet = {}) { - this.tools = tools + constructor(availableTools: ToolDefinitionSet = {}) { + this.updateTools(availableTools) } async prepare(): Promise {} @@ -265,20 +265,24 @@ class AppleLLMChatLanguageModel implements LanguageModelV3 { private prepareTools(tools: Tool[] = []) { return tools.map((tool) => { if (tool.type === 'function') { - const toolDefinition = this.tools[tool.name] - if (!toolDefinition) { - throw new Error(`Tool ${tool.name} not found`) - } const schema = jsonSchema(tool.inputSchema) return { ...tool, id: generateId(), - execute: async (modelInput: any, opts: ToolCallOptions) => { + execute: async (modelInput: unknown) => { + const text = + typeof modelInput === 'string' + ? modelInput + : JSON.stringify(modelInput ?? '') const toolCallArguments = await parseJSON({ - text: modelInput, + text, schema, }) - return toolDefinition.execute?.(toolCallArguments, opts) + const opts: ToolCallOptions = { + toolCallId: generateId(), + messages: [], + } + return this.tools[tool.name].execute?.(toolCallArguments, opts) }, } } @@ -286,6 +290,10 @@ class AppleLLMChatLanguageModel implements LanguageModelV3 { }) } + updateTools(tools: ToolDefinitionSet) { + this.tools = tools + } + async doGenerate(options: LanguageModelV3CallOptions) { const messages = this.prepareMessages(options.prompt) const tools = this.prepareTools(options.tools) @@ -403,17 +411,24 @@ class AppleLLMChatLanguageModel implements LanguageModelV3 { id: streamId, }) - let previousContent = '' + let previousRawContent = '' const updateListener = NativeAppleLLM.onStreamUpdate((data) => { if (data.streamId === streamId) { - const delta = data.content.slice(previousContent.length) + const nextRawContent = String(data.content ?? '') + const rawDelta = nextRawContent.startsWith(previousRawContent) + ? nextRawContent.slice(previousRawContent.length) + : nextRawContent + previousRawContent = nextRawContent + + // Apple native streaming can emit bogus "null" chunks as text. + if (rawDelta === 'null') return + controller.enqueue({ type: 'text-delta', - delta, + delta: rawDelta, id: data.streamId, }) - previousContent = data.content } }) diff --git a/packages/json-ui/README.md b/packages/json-ui/README.md new file mode 100644 index 00000000..a55bd291 --- /dev/null +++ b/packages/json-ui/README.md @@ -0,0 +1,226 @@ +# `@react-native-ai/json-ui` + +Lightweight JSON UI tooling for React Native + Vercel AI SDK tool calling. + +**This package provides:** + +- a component/style registry (`GEN_UI_NODE_HINTS`, `GEN_UI_STYLES`, `GEN_UI_NODE_NAMES`) +- a ready-to-use tool set for JSON UI mutation (`createGenUITools`) +- a reusable system prompt builder (`buildGenUISystemPrompt`) +- a React Native renderer for specs (`GenerativeUIView`), which passes `GEN_UI_STYLES` (overridable) to the default `GenUINode` and lets you supply a custom node renderer + +**Why this package?** + +There exists a great library for streaming interfaces: [`json-render`](https://github.com/vercel-labs/json-render). The full specification is provided in the ['Prior art'](#prior-art) section, but **TL;DR**: this library - `@react-native-ai/json-ui` - is the choice for small language models (e.g. parameters in the order of magnitude of 3B), which is usually the case if you are running inference locally, on-device. If you are using a cloud provider, consider `json-render` instead. + +## Requirements + +- React Native app +- Vercel AI SDK tool-calling flow (`streamText`, `generateText`, etc.) +- model that supports tool calling + +## Installation + +```sh +bun add @react-native-ai/json-ui +``` + +## Quick Start + +```ts +import { streamText } from 'ai' +import { + buildGenUISystemPrompt, + createGenUITools, +} from '@react-native-ai/json-ui' + +const tools = createGenUITools({ + contextId: chatId, + getSpec: (id) => getSpecForChat(id), + updateSpec: (id, nextSpec) => setSpecForChat(id, nextSpec), + toolWrapper: (toolName, execute) => async (args) => { + console.log('Executing tool', toolName, args) + return execute(args) + }, +}) + +const result = streamText({ + model, + messages, + tools, + system: buildGenUISystemPrompt({ + additionalInstructions: + 'Your name is John. Keep responses short. Ask follow-up questions when UI intent is unclear.', + }), +}) +``` + +## API + +### `createGenUITools(options)` + +Creates a set of tool definitions: + +- `getUIRootNode` +- `getUINode` +- `getUILayout` +- `getAvailableUINodes` +- `setUINodeProps` +- `deleteUINode` +- `addUINode` +- `reorderUINodes` + +`reorderUINodes` moves one sibling by index offset (with clamping): + +- `nodeId`: node to move +- `offset`: integer shift among siblings (`< 0` moves earlier, `> 0` moves later) + +Options: + +- `contextId`: string key for your current conversation/context +- `getSpec(contextId)`: return current spec (or `null`) +- `updateSpec(contextId, spec)`: persist changes +- `createId`: optional id factory +- `rootId`: optional root id override (default: `"root"`) +- `nodeHints`: optional component registry override +- `nodeNamesThatSupportChildren`: optional parent whitelist override +- `toolWrapper(toolName, execute)`: required wrapper used to decorate all tool executions (for logging, error handling, telemetry, etc.) + +Tool behavior details: + +- `setUINodeProps`: `{ id, props, replace? }`; defaults to merge mode, set `replace: true` to replace all props +- `addUINode`: `{ parentId?, type, props? }`; if `parentId` is omitted it defaults to root +- `reorderUINodes`: clamps index movement to valid sibling bounds + +### `buildGenUISystemPrompt(options?)` + +Builds the reusable, non-app-specific system instructions for JSON UI tooling. + +Options: + +- `additionalInstructions`: append app-specific guidance +- `requireLayoutReadBeforeAddingNodes`: defaults to `true` +- `styleHints`: override style metadata used in prompt text + +### Registries + +- `GEN_UI_NODE_NAMES` +- `GEN_UI_NODE_HINTS` +- `GEN_UI_NODE_NAMES_THAT_SUPPORT_CHILDREN` +- `GEN_UI_STYLES` +- `GEN_UI_STYLE_HINTS` + +### `GenerativeUIView` + +Renders a JSON UI spec directly in React Native. The default node renderer (`GenUINode`) receives style validators from the view; you can override styles or supply a custom renderer. + +```tsx +import { GenerativeUIView } from '@react-native-ai/json-ui' +; +``` + +Props: + +- `spec`: `{ root, elements }` object (or `null`/`undefined`) +- `loading`: optional boolean for empty-state loading text +- `showCollapsibleJSON`: optional boolean to render an expandable JSON debug panel +- `styles`: optional override for style validators (merged with default `GEN_UI_STYLES`); the resulting map is passed to `GenUINode` +- `GenUINodeComponent`: optional custom component to render the tree; receives `{ nodeId, elements, styles }`; delegate to default `GenUINode` for nodes you don’t handle + +#### Styles from GenerativeUIView + +The default `GenUINode` does not import `GEN_UI_STYLES` itself. `GenerativeUIView` merges the registry default with any `styles` prop and passes the result down as the `styles` prop to `GenUINode`. So all style validation is driven by what the view provides, and you can pass custom validators: + +```tsx +import { z } from 'zod' +import { GenerativeUIView, GEN_UI_STYLES } from '@react-native-ai/json-ui' +; +``` + +#### Custom GenUINode example + +You can supply your own node renderer and reuse the library’s style/prop parsing for custom component types. For one type render a custom component using `parseGenUIElementProps`; for everything else use the default `GenUINode`: + +```tsx +import { View, Text, StyleSheet } from 'react-native' +import { + GenerativeUIView, + GenUINode, + parseGenUIElementProps, + type GenUINodeProps, +} from '@react-native-ai/json-ui' + +const BADGE_TYPE = 'Badge' + +function CustomGenUINode({ + nodeId, + elements, + styles, + GenUINodeComponent, +}: GenUINodeProps) { + const element = elements[nodeId] + if (!element) return null + if (element.type === BADGE_TYPE) { + const { baseStyle, text } = parseGenUIElementProps(element, styles, { + nodeId, + type: BADGE_TYPE, + }) + return ( + + {text ?? ''} + + ) + } + return ( + + ) +} + +const customStyles = StyleSheet.create({ + badge: { alignSelf: 'flex-start', paddingHorizontal: 8, paddingVertical: 4, borderRadius: 12 }, + badgeText: { fontSize: 12, color: '#fff' }, +}) + +// Use the custom renderer; styles still come from the view (default or overridden). + +``` + +For a full usage example, see [this file](https://github.com/callstackincubator/ai/tree/main/generative-ui/apps/expo-example/src/store/chatStore.ts). + +## Notes + +- style validation is schema-based (`zod`) through `GEN_UI_STYLES` +- tools are framework-agnostic as long as your spec matches `{ root, elements }` +- this package is intentionally minimal and optimized for small-model tool loops + +## Prior art + +[`json-render`](https://github.com/vercel-labs/json-render) is an alternative that has existed before this library was introduced. +It provides a package both for React and React Native integration and is also compatible with the [AI SDK](https://github.com/vercel/ai). Its design involves two primary concepts: + +- the LLM streams the UI directly in a predefined format and all outputs are intercepted by the stream parser +- the library provides a (long) system prompt which well describes that format for the LLM to follow, along with examples + +That works well for **large language models** (e.g. cloud APIs). For **on-device, small models** (e.g. Apple Foundation Models with limited context), you run into: + +1. **Context size** — Small models often have 4K-token windows (such as Apple Foundation having a 4096 token limit). A long system prompt plus conversation leaves little room; you end up summarizing or truncating, if you even fit into the window at all. +2. **Task complexity** — json-render supports rich actions and state. For small models (e.g. 3B parameters), generating a correct static UI is already hard; a simpler, tool-based flow is more reliable. + +How this library differs is: + +- **Tool calling instead of streaming UI** — The model emits small JSON payloads by calling tools (add node, set props, etc.). Each step is small and easier for the model to get right. +- **Narrower feature set** — Focus on static UI building first, so smaller models can complete the task. More features will be added later. + +## License + +MIT diff --git a/packages/json-ui/babel.config.js b/packages/json-ui/babel.config.js new file mode 100644 index 00000000..3e0218e6 --- /dev/null +++ b/packages/json-ui/babel.config.js @@ -0,0 +1,3 @@ +module.exports = { + presets: ['module:@react-native/babel-preset'], +} diff --git a/packages/json-ui/package.json b/packages/json-ui/package.json new file mode 100644 index 00000000..90204eea --- /dev/null +++ b/packages/json-ui/package.json @@ -0,0 +1,56 @@ +{ + "name": "@react-native-ai/json-ui", + "version": "1.0.0-alpha.1", + "description": "Lightweight JSON UI generation for React Native based on tool calling, compatible with Vercel AI SDK", + "main": "lib/commonjs/index", + "module": "lib/module/index", + "types": "lib/typescript/index.d.ts", + "react-native": "src/index", + "source": "src/index", + "files": [ + "src", + "lib", + "!**/__tests__", + "!**/__fixtures__", + "!**/__mocks__", + "!**/.*", + "README.md" + ], + "author": "artus9033 ", + "scripts": { + "clean": "del-cli lib", + "typecheck": "tsc --noEmit", + "prepare": "bob build" + }, + "dependencies": { + "ai": "^6.0.0", + "zod": "^4.0.0" + }, + "peerDependencies": { + "react": "*", + "react-native": "*" + }, + "react-native-builder-bob": { + "source": "src", + "output": "lib", + "targets": [ + "commonjs", + "module", + [ + "typescript", + { + "project": "tsconfig.build.json" + } + ] + ] + }, + "keywords": [ + "react-native", + "ui", + "generative", + "llm", + "ai", + "sdk", + "on-device" + ] +} diff --git a/packages/json-ui/src/GenUINode.tsx b/packages/json-ui/src/GenUINode.tsx new file mode 100644 index 00000000..26fc7866 --- /dev/null +++ b/packages/json-ui/src/GenUINode.tsx @@ -0,0 +1,115 @@ +import React from 'react' +import { Pressable, StyleSheet, Text, TextInput, View } from 'react-native' + +import type { GenUIStylesConfig } from './parseGenUIProps' +import { parseGenUIElementProps } from './parseGenUIProps' +import { GEN_UI_NODE_NAMES, type JsonUISpec } from './registry' + +export type GenUINodeProps = { + nodeId: string + elements: JsonUISpec['elements'] + styles: GenUIStylesConfig + /** When provided by GenerativeUIView, used to render children so custom types are handled at any depth. */ + GenUINodeComponent?: React.ComponentType +} + +export function GenUINode({ + nodeId, + elements, + styles: styleValidators, + GenUINodeComponent, +}: GenUINodeProps) { + const element = elements[nodeId] + if (!element) return null + + const { type, props, children = [] } = element + const { baseStyle, text, label } = parseGenUIElementProps( + element, + styleValidators, + { nodeId, type } + ) + + const ChildRenderer = GenUINodeComponent ?? GenUINode + const childProps: Omit = { + elements, + styles: styleValidators, + GenUINodeComponent, + } + + switch (type) { + case 'Container': + return ( + + {children.map((id) => ( + + ))} + + ) + + case GEN_UI_NODE_NAMES.Text: + case GEN_UI_NODE_NAMES.Paragraph: + case GEN_UI_NODE_NAMES.Label: + case GEN_UI_NODE_NAMES.Heading: + return {text ?? label ?? ''} + + case GEN_UI_NODE_NAMES.Button: + return ( + + + {label ?? text ?? ''} + + + ) + + case GEN_UI_NODE_NAMES.TextInput: + return ( + {}} + /> + ) + + default: + return ( + + {children.map((id) => ( + + ))} + + ) + } +} + +const styles = StyleSheet.create({ + container: { + gap: 8, + flex: 1, + }, + text: { + fontSize: 16, + color: '#111827', + flex: 1, + }, + button: { + paddingHorizontal: 16, + paddingVertical: 10, + backgroundColor: '#3b82f6', + borderRadius: 8, + alignSelf: 'flex-start', + flex: 1, + }, + buttonText: { + flex: 1, + color: 'white', + }, + textInput: { + borderWidth: 1, + borderColor: '#e0e0e0', + borderRadius: 8, + padding: 8, + fontSize: 16, + color: '#111827', + flex: 1, + }, +}) diff --git a/packages/json-ui/src/GenerativeUIView.tsx b/packages/json-ui/src/GenerativeUIView.tsx new file mode 100644 index 00000000..8bde3024 --- /dev/null +++ b/packages/json-ui/src/GenerativeUIView.tsx @@ -0,0 +1,137 @@ +import React from 'react' +import { Pressable, StyleSheet, Text, View } from 'react-native' + +import type { GenUINodeProps } from './GenUINode' +import { GenUINode } from './GenUINode' +import type { GenUIStylesConfig } from './parseGenUIProps' +import { GEN_UI_STYLES, type JsonUISpec } from './registry' + +export type GenerativeUIViewProps = { + /** Normalized spec (root + elements with root node "root"). */ + spec: JsonUISpec | null | undefined + loading?: boolean + /** Show expandable JSON payload for the current UI spec. */ + showCollapsibleJSON?: boolean + /** + * Override or extend style validators (zod schemas) for node props. + * Merged with default GEN_UI_STYLES and passed to GenUINode. + */ + styles?: Partial & GenUIStylesConfig + /** + * Custom node renderer. Receives nodeId, elements, styles (same as default GenUINode). + * Use for custom component types; delegate to default GenUINode for others. + */ + GenUINodeComponent?: React.ComponentType +} + +export function GenerativeUIView({ + spec, + loading, + showCollapsibleJSON, + styles: stylesOverride, + GenUINodeComponent, +}: GenerativeUIViewProps) { + const [expanded, setExpanded] = React.useState(false) + const normalized = React.useMemo(() => { + if (!spec?.root || !spec.elements) return null + const rootElement = spec.elements[spec.root] + if (!rootElement) return null + return spec + }, [spec]) + + const styleValidators: GenUIStylesConfig = React.useMemo( + () => ({ ...GEN_UI_STYLES, ...stylesOverride }), + [stylesOverride] + ) + + if (!normalized) { + return ( + + + {loading + ? 'Loading…' + : 'Use tools getGenUIRootNode, addNode, etc. to build the UI.'} + + + ) + } + + const NodeRenderer = GenUINodeComponent ?? GenUINode + return ( + + + {showCollapsibleJSON ? ( + + setExpanded((prev) => !prev)} + style={styles.jsonHeader} + > + UI JSON + + {expanded ? '[-]' : '[+]'} + + + {expanded ? ( + + {JSON.stringify(normalized, null, 2)} + + ) : null} + + ) : null} + + ) +} + +const styles = StyleSheet.create({ + wrapper: { + flex: 1, + justifyContent: 'center', + gap: 8, + }, + jsonPanel: { + borderWidth: 1, + borderColor: '#e5e7eb', + borderRadius: 12, + paddingHorizontal: 12, + paddingVertical: 10, + backgroundColor: '#f9fafb', + }, + jsonHeader: { + flexDirection: 'row', + justifyContent: 'space-between', + alignItems: 'center', + gap: 8, + }, + jsonHeaderText: { + fontSize: 13, + color: '#4b5563', + fontWeight: '600', + }, + jsonHeaderIcon: { + fontSize: 12, + color: '#9ca3af', + fontWeight: '600', + }, + jsonText: { + marginTop: 8, + fontSize: 12, + lineHeight: 18, + color: '#374151', + fontFamily: 'Menlo', + }, + placeholder: { + flex: 1, + justifyContent: 'center', + padding: 24, + }, + placeholderText: { + fontSize: 15, + color: '#6b7280', + textAlign: 'center', + }, +}) diff --git a/packages/json-ui/src/index.ts b/packages/json-ui/src/index.ts new file mode 100644 index 00000000..ba256611 --- /dev/null +++ b/packages/json-ui/src/index.ts @@ -0,0 +1,25 @@ +export { + GenerativeUIView, + type GenerativeUIViewProps, +} from './GenerativeUIView' +export { GenUINode, type GenUINodeProps } from './GenUINode' +export { + type GenUIStylesConfig, + type ParsedGenUIProps, + parseGenUIElementProps, + type ParseGenUIElementPropsOptions, +} from './parseGenUIProps' +export { + buildGenUISystemPrompt, + type BuildGenUISystemPromptOptions, +} from './prompt' +export type { JsonUIElement, JsonUISpec } from './registry' +export { + DEFAULT_GEN_UI_ROOT_ID, + GEN_UI_NODE_HINTS, + GEN_UI_NODE_NAMES, + GEN_UI_NODE_NAMES_THAT_SUPPORT_CHILDREN, + GEN_UI_STYLE_HINTS, + GEN_UI_STYLES, +} from './registry' +export { type CreateGenTUIoolsOptions, createGenUITools } from './tools' diff --git a/packages/json-ui/src/parseGenUIProps.ts b/packages/json-ui/src/parseGenUIProps.ts new file mode 100644 index 00000000..13326aac --- /dev/null +++ b/packages/json-ui/src/parseGenUIProps.ts @@ -0,0 +1,59 @@ +import type { z } from 'zod' + +import type { JsonUIElement } from './registry' + +export type GenUIStylesConfig = Record + +export type ParsedGenUIProps = { + baseStyle: Record + text: string | undefined + label: string | undefined + props: Record +} + +export type ParseGenUIElementPropsOptions = { + nodeId?: string + type?: string +} + +/** + * Parses a JSON UI element's props into a baseStyle object (validated by style validators) + * and common fields (text, label). Use this in custom GenUINode implementations to reuse + * the same style and prop parsing as the default renderer. + */ +export function parseGenUIElementProps( + element: JsonUIElement, + styleValidators: GenUIStylesConfig, + options?: ParseGenUIElementPropsOptions +): ParsedGenUIProps { + const { nodeId = '', type = '' } = options ?? {} + const props = element.props ?? {} + const style = props.style as Record | undefined + const flex = props.flex as number | undefined + const padding = props.padding as number | undefined + const gap = props.gap as number | undefined + const text = (props.text ?? props.value ?? props.label) as string | undefined + const label = (props.label ?? props.text) as string | undefined + + const baseStyle = { + ...(flex != null ? { flex } : {}), + ...(padding != null ? { padding } : {}), + ...(gap != null ? { gap } : {}), + ...style, + } as Record + + for (const key of Object.keys(props)) { + const validator = styleValidators[key] + if (validator) { + if (validator.safeParse(props[key]).success) { + baseStyle[key] = props[key] + } else { + console.warn( + `[@react-native-ai/json-ui] Invalid style prop: ${key} for node ${nodeId} of type ${type}: ${props[key]}` + ) + } + } + } + + return { baseStyle, text, label, props } +} diff --git a/packages/json-ui/src/prompt.ts b/packages/json-ui/src/prompt.ts new file mode 100644 index 00000000..6510dec7 --- /dev/null +++ b/packages/json-ui/src/prompt.ts @@ -0,0 +1,49 @@ +import { GEN_UI_STYLE_HINTS } from './registry' + +type StyleHints = Record< + string, + { + type: 'number' | 'string' + description?: string + } +> + +export type BuildGenUISystemPromptOptions = { + additionalInstructions?: string + requireLayoutReadBeforeAddingNodes?: boolean + styleHints?: StyleHints +} + +export function buildGenUISystemPrompt({ + additionalInstructions, + requireLayoutReadBeforeAddingNodes = true, + styleHints = GEN_UI_STYLE_HINTS, +}: BuildGenUISystemPromptOptions = {}) { + const styleKeysText = Object.entries(styleHints) + .map(([key, entry]) => { + let text = `${key} [${entry.type}]` + if (entry.description) text += ` (${entry.description})` + return text + }) + .join(', ') + + const parts = [ + 'You are a helpful assistant.', + 'You have tools to create and update UI nodes. Before any tool calls for UI, ALWAYS CALL getUILayout before and after.', + 'Remember this is React Native, not web, and use simple props.', + `If you set the "style" prop on a UI node, the possible keys are: ${styleKeysText}.`, + 'Remember NEVER use web values.', + ] + + if (requireLayoutReadBeforeAddingNodes) { + parts.push( + 'BEFORE ADDING ANY UI ELEMENTS, GET THE WHOLE UI TREE with getGenUILayout.' + ) + } + + if (additionalInstructions?.trim()) { + parts.push(additionalInstructions.trim()) + } + + return parts.join(' ') +} diff --git a/packages/json-ui/src/registry.ts b/packages/json-ui/src/registry.ts new file mode 100644 index 00000000..718f5114 --- /dev/null +++ b/packages/json-ui/src/registry.ts @@ -0,0 +1,73 @@ +import { z } from 'zod' + +export type JsonUIElement = { + type: string + props: Record + children?: string[] +} + +export type JsonUISpec = { + root: string + elements: Record +} + +export const DEFAULT_GEN_UI_ROOT_ID = 'root' + +export const GEN_UI_NODE_NAMES = { + Text: 'Text', + Paragraph: 'Paragraph', + Label: 'Label', + Heading: 'Heading', + Button: 'Button', + TextInput: 'TextInput', +} as const + +export const GEN_UI_NODE_NAMES_THAT_SUPPORT_CHILDREN: string[] = [] + +export const GEN_UI_NODE_HINTS: Record = + { + Text: 'Single line text. Props: text [string], style [object].', + Paragraph: 'Long text. Props: text [string], style [object].', + Label: 'Small label. Props: text [string], style [object].', + Heading: 'Title text. Props: text [string], style [object].', + Button: 'Tap button. Props: text [string], style [object].', + TextInput: + 'Single line text input. Props: placeholder [string], style [object].', + } + +export const GEN_UI_STYLES = { + flex: z.number(), + padding: z.number(), + gap: z.number(), + backgroundColor: z.string(), + color: z.string(), + fontSize: z.number(), + fontWeight: z.string(), + textAlign: z.string(), +} + +export const GEN_UI_STYLE_HINTS: Record< + keyof typeof GEN_UI_STYLES, + { type: 'number' | 'string'; description?: string } +> = { + flex: { type: 'number', description: 'Flex grow/shrink basis value.' }, + padding: { + type: 'number', + description: 'Padding in density-independent px.', + }, + gap: { + type: 'number', + description: 'Spacing between children in density-independent px.', + }, + backgroundColor: { type: 'string', description: 'React Native color value.' }, + color: { type: 'string', description: 'Text color value.' }, + fontSize: { type: 'number', description: 'Font size in points.' }, + fontWeight: { + type: 'string', + description: 'Font weight string like "400", "600", "bold".', + }, + textAlign: { + type: 'string', + description: 'Text alignment, for example "left", "center", "right".', + }, +} diff --git a/packages/json-ui/src/tools.ts b/packages/json-ui/src/tools.ts new file mode 100644 index 00000000..241c0707 --- /dev/null +++ b/packages/json-ui/src/tools.ts @@ -0,0 +1,394 @@ +import { tool } from 'ai' +import { z } from 'zod' + +import { + DEFAULT_GEN_UI_ROOT_ID, + GEN_UI_NODE_HINTS, + GEN_UI_NODE_NAMES_THAT_SUPPORT_CHILDREN, + type JsonUISpec, +} from './registry' + +/** + * Sometimes LLMs call tools with a string instead of an object. + */ +function smartParse( + props: string | Record +): Record { + return typeof props === 'string' ? JSON.parse(props) : props +} + +const defaultCreateId = () => + `UI-${Date.now().toString(36)}-${Math.random().toString(16).slice(2)}` + +export type CreateGenTUIoolsOptions = { + contextId: string + getSpec: (contextId: string) => TSpec | null + updateSpec: (contextId: string, spec: TSpec | null) => void + createId?: () => string + rootId?: string + nodeHints?: Record + nodeNamesThatSupportChildren?: readonly string[] + toolWrapper?: ( + toolName: string, + execute: (args: TArgs) => Promise + ) => (args: TArgs) => Promise +} + +const cloneSpec = (spec: TSpec): TSpec => + ({ + ...spec, + elements: Object.fromEntries( + Object.entries(spec.elements).map(([id, element]) => [ + id, + { + ...element, + props: { ...(element.props ?? {}) }, + children: [...(element.children ?? [])], + }, + ]) + ), + }) as TSpec + +let mutationQueue: Promise = Promise.resolve() + +const withMutationLock = async (run: () => Promise): Promise => { + let release: () => void = () => {} + const pending = new Promise((resolve) => { + release = resolve + }) + const previous = mutationQueue + mutationQueue = mutationQueue.then(() => pending) + + await previous + try { + return await run() + } finally { + release() + } +} + +/** + * Creates generative UI tools that read/update a JSON UI spec. + */ +export function createGenUITools({ + contextId, + getSpec, + updateSpec, + createId = defaultCreateId, + rootId = DEFAULT_GEN_UI_ROOT_ID, + nodeHints = GEN_UI_NODE_HINTS, + nodeNamesThatSupportChildren = GEN_UI_NODE_NAMES_THAT_SUPPORT_CHILDREN, + toolWrapper = (_, execute) => execute, +}: CreateGenTUIoolsOptions) { + // Serialize mutating tool calls to avoid interleaving writes. + let cachedSpec: TSpec | null = null + + const readSpec = (): TSpec | null => { + if (cachedSpec) return cloneSpec(cachedSpec) + const spec = getSpec(contextId) + if (!spec) return null + cachedSpec = cloneSpec(spec) + return cloneSpec(cachedSpec) + } + + const commitSpec = (spec: TSpec | null) => { + cachedSpec = spec ? cloneSpec(spec) : null + updateSpec(contextId, spec ? cloneSpec(spec) : null) + } + + const getUIRootNode = tool({ + description: + 'Get the root node of the generative UI tree. Returns id, type, props, and children (array of { id, type }). Root always exists with id "root".', + inputSchema: z.object({}), + execute: toolWrapper('getUIRootNode', async () => { + const spec = readSpec() + if (!spec?.root || !spec.elements[spec.root]) return { root: null } + const element = spec.elements[spec.root] + const children = (element.children ?? []).map((id) => ({ + id, + type: spec.elements[id]?.type ?? 'unknown', + })) + return { + root: { + id: spec.root, + type: element.type, + props: element.props, + children, + }, + } + }), + }) + + const getUINode = tool({ + description: + 'Get a node by id. Returns id, type, props, and children (array of { id, type }). If id is omitted, returns root node.', + inputSchema: z.object({ + id: z.string().optional().describe('Node id; omit for root'), + }), + execute: toolWrapper('getUINode', async ({ id }) => { + const spec = readSpec() + if (!spec) return { node: null } + const nodeId = id ?? spec.root + const element = spec.elements[nodeId] + if (!element) return { node: null } + const children = (element.children ?? []).map((childId) => ({ + id: childId, + type: spec.elements[childId]?.type ?? 'unknown', + })) + return { + node: { + id: nodeId, + type: element.type, + props: element.props, + children, + }, + } + }), + }) + + const getUILayout = tool({ + description: 'Get compact UI layout.', + inputSchema: z.object({}), + execute: toolWrapper('getUILayout', async () => { + const spec = readSpec() + if (!spec) return { root: null, nodes: [] } + + const parentByChild: Record = {} + for (const [id, element] of Object.entries(spec.elements)) { + for (const childId of element.children ?? []) { + parentByChild[childId] = id + } + } + + const nodes = Object.entries(spec.elements).map(([id, element]) => ({ + id, + type: element.type, + parentId: parentByChild[id] ?? null, + children: element.children ?? [], + props: Object.keys(element.props ?? {}), + })) + + return { root: spec.root, nodes } + }), + }) + + const getAvailableUINodes = tool({ + description: 'List nodes + props.', + inputSchema: z.object({}), + execute: toolWrapper('getAvailableUINodes', async () => ({ + nodes: Object.entries(nodeHints).map(([name, props]) => ({ + name, + props, + })), + })), + }) + + const setUINodeProps = tool({ + description: 'Set or add props for a node by id.', + inputSchema: z.object({ + id: z.string().describe('Node id'), + props: z.string().describe('Props object for the node'), + replace: z.boolean().optional().describe('Replace existing props'), + }), + execute: toolWrapper( + 'setUINodeProps', + async ({ id, props: propsArg, replace = false }) => + withMutationLock(async () => { + const parsedProps = smartParse(propsArg) + const spec = readSpec() + if (!spec) return { success: false, message: 'No UI spec' } + if (!spec.elements[id]) { + return { success: false, message: 'Node not found' } + } + + const elements = { ...spec.elements } + const current = elements[id] + const nextProps = replace + ? parsedProps + : { ...current.props, ...parsedProps } + + elements[id] = { ...current, props: nextProps } + + commitSpec({ root: spec.root, elements } as TSpec) + return { success: true } + }) + ), + }) + + const deleteUINode = tool({ + description: + 'Delete a node by id. Cannot delete the root node (id "root"). Removes the node and its reference from the parent\'s children.', + inputSchema: z.object({ + id: z.string().describe('Node id to delete'), + }), + execute: toolWrapper('deleteUINode', async ({ id }) => + withMutationLock(async () => { + if (id === rootId) { + return { success: false, message: 'Cannot delete root node' } + } + const spec = readSpec() + if (!spec) return { success: false, message: 'No UI spec' } + const elements = { ...spec.elements } + delete elements[id] + for (const key of Object.keys(elements)) { + const element = elements[key] + if (element.children?.includes(id)) { + elements[key] = { + ...element, + children: element.children.filter((childId) => childId !== id), + } + } + } + commitSpec({ root: spec.root, elements } as TSpec) + return { success: true } + }) + ), + }) + + const addUINode = tool({ + description: + 'Add a new node as a child of parentId. Creates element with type and props. Returns new node id. Props must be a valid JSON object.', + inputSchema: z.object({ + parentId: z.string().optional().describe('Parent node id; omit for root'), + type: z + .string() + .describe('Component type (e.g. Container, Column, Text, Button)'), + props: z.string().optional().describe('Props object for the node'), + }), + execute: toolWrapper( + 'addUINode', + async ({ parentId, type, props: propsArg }) => + withMutationLock(async () => { + const parsedProps = smartParse(propsArg ?? '{}') + const spec = readSpec() + if (!spec) { + console.warn( + '[@react-native-ai/json-ui tool addNode] No UI spec, aborting' + ) + + return { success: false, message: 'No UI spec' } + } + + parentId ??= spec.root + + if (!spec.elements[parentId]) { + console.warn( + '[@react-native-ai/json-ui tool addNode] Parent not found, aborting' + ) + return { success: false, message: 'Parent not found' } + } + + const newId = createId() + spec.elements[newId] = { + type, + props: parsedProps ?? {}, + children: [], + } + let parent = spec.elements[parentId] + + if (!nodeNamesThatSupportChildren.includes(parent.type)) { + parent = spec.elements[spec.root] + parentId = spec.root + } + + spec.elements[parentId].children ??= [] + spec.elements[parentId].children!.push(newId) + + commitSpec({ + root: spec.root, + elements: spec.elements, + } as TSpec) + return { success: true, id: newId } + }) + ), + }) + + const reorderUINodes = tool({ + description: + 'Move one node among siblings by offset (negative = up, positive = down).', + inputSchema: z.object({ + nodeId: z.string().describe('Node id to move'), + offset: z + .number() + .describe( + 'Relative index shift among siblings; negative moves earlier, positive moves later' + ), + }), + execute: toolWrapper('reorderUINodes', async ({ nodeId, offset }) => + withMutationLock(async () => { + const spec = readSpec() + if (!spec) return { success: false, message: 'No UI spec' } + + const findParentId = (childId: string) => { + for (const [id, element] of Object.entries(spec.elements)) { + if (element.children?.includes(childId)) return id + } + return null + } + + const nodeParentId = findParentId(nodeId) + if (!nodeParentId) { + return { + success: false, + message: 'nodeId must exist and have a parent', + } + } + + const parentId = nodeParentId + const parent = spec.elements[parentId] + if (!parent) return { success: false, message: 'Parent not found' } + + const currentChildren = [...(parent.children ?? [])] + const nodeIndex = currentChildren.indexOf(nodeId) + if (nodeIndex === -1) { + return { + success: false, + message: 'nodeId must be a direct child', + } + } + + if (offset === 0) { + return { + success: true, + parentId, + nodeId, + fromIndex: nodeIndex, + toIndex: nodeIndex, + appliedOffset: 0, + childIds: currentChildren, + } + } + + const maxIndex = currentChildren.length - 1 + const toIndex = Math.min(Math.max(nodeIndex + offset, 0), maxIndex) + currentChildren.splice(nodeIndex, 1) + currentChildren.splice(toIndex, 0, nodeId) + + const elements = { ...spec.elements } + elements[parentId].children = currentChildren + commitSpec({ root: spec.root, elements } as TSpec) + + return { + success: true, + parentId, + nodeId, + fromIndex: nodeIndex, + toIndex, + appliedOffset: toIndex - nodeIndex, + childIds: currentChildren, + } + }) + ), + }) + + return { + getUIRootNode, + getUINode, + getUILayout, + getAvailableUINodes, + setUINodeProps, + deleteUINode, + addUINode, + reorderUINodes, + } +} diff --git a/packages/json-ui/tsconfig.build.json b/packages/json-ui/tsconfig.build.json new file mode 100644 index 00000000..acbe349f --- /dev/null +++ b/packages/json-ui/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig", + "include": ["src/**/*"], + "exclude": ["**/__tests__/**"] +} diff --git a/packages/json-ui/tsconfig.json b/packages/json-ui/tsconfig.json new file mode 100644 index 00000000..bf5a36dd --- /dev/null +++ b/packages/json-ui/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src/**/*"] +} diff --git a/packages/mlc/ios/MLCEngine.mm b/packages/mlc/ios/MLCEngine.mm index 0b58c3bf..099a7a03 100644 --- a/packages/mlc/ios/MLCEngine.mm +++ b/packages/mlc/ios/MLCEngine.mm @@ -405,7 +405,6 @@ - (void)prepareModel:(NSString *)modelId } [self.engine reload:engineConfigJSON]; - resolve([NSString stringWithFormat:@"Model prepared: %@", modelId]); } @catch (NSException *exception) { reject(@"MLCEngine", exception.reason, nil); diff --git a/website/src/docs/_meta.json b/website/src/docs/_meta.json index ab41b941..e30bf51a 100644 --- a/website/src/docs/_meta.json +++ b/website/src/docs/_meta.json @@ -34,5 +34,12 @@ "label": "MLC", "collapsible": true, "collapsed": false + }, + { + "type": "dir", + "name": "json-ui", + "label": "JSON UI", + "collapsible": true, + "collapsed": false } ] diff --git a/website/src/docs/apple/generating.md b/website/src/docs/apple/generating.md index 0b0e6471..815edb02 100644 --- a/website/src/docs/apple/generating.md +++ b/website/src/docs/apple/generating.md @@ -102,6 +102,22 @@ const apple = createAppleProvider({ }); ``` +If you want to change the tools at runtime, you can do it as follows: + +```typescript +const apple = createAppleProvider({ + availableTools: { + getWeather + } +}); +const model = apple(); + +model.updateTools({ + getWeather, + getDate +}); +``` + ### Basic Tool Usage Then, generate output like with any other Vercel AI SDK provider: diff --git a/website/src/docs/index.md b/website/src/docs/index.md index e6e7ca2c..07dd6091 100644 --- a/website/src/docs/index.md +++ b/website/src/docs/index.md @@ -5,7 +5,7 @@ A collection of on-device AI primitives for React Native with first-class Vercel ## Why On-Device AI? - **Privacy-first:** All processing happens locally—no data leaves the device -- **Instant responses:** No network latency, immediate AI capabilities +- **Instant responses:** No network latency, immediate AI capabilities - **Offline-ready:** Works anywhere, even without internet - **Zero server costs:** No API fees or infrastructure to maintain @@ -50,6 +50,16 @@ Run any open-source LLM locally using MLC's optimized runtime through `@react-na > [!NOTE] > MLC support is experimental and not recommended for production use yet. +### JSON UI + +Build UIs from tool-calling models with `@react-native-ai/json-ui`: + +- **Tool-based spec** — Model calls tools to add/set/delete nodes and props +- **GenerativeUIView** — Renders the spec in React Native; override styles or supply a custom node renderer +- **Small-model friendly** — Designed for on-device models with limited context + +See the [JSON UI docs](./json-ui/getting-started) for setup and API. + ### Google (Coming Soon) Support for Google's on-device models is planned for future releases. diff --git a/website/src/docs/json-ui/_meta.json b/website/src/docs/json-ui/_meta.json new file mode 100644 index 00000000..7fa1158d --- /dev/null +++ b/website/src/docs/json-ui/_meta.json @@ -0,0 +1,22 @@ +[ + { + "type": "file", + "name": "getting-started", + "label": "Getting Started" + }, + { + "type": "file", + "name": "tools", + "label": "Tools" + }, + { + "type": "file", + "name": "view", + "label": "GenUI View component" + }, + { + "type": "file", + "name": "prior-art", + "label": "Prior Art" + } +] diff --git a/website/src/docs/json-ui/getting-started.md b/website/src/docs/json-ui/getting-started.md new file mode 100644 index 00000000..b7e5bf6c --- /dev/null +++ b/website/src/docs/json-ui/getting-started.md @@ -0,0 +1,79 @@ +# Getting Started + +Lightweight JSON UI tooling for React Native with the Vercel AI SDK. The model builds and updates a UI by calling tools (e.g. add node, set props); you render the resulting spec with `GenerativeUIView`. + +## What this package provides + +- **Component/style registry** — `GEN_UI_NODE_HINTS`, `GEN_UI_STYLES`, `GEN_UI_NODE_NAMES` for tools and prompts +- **Tool set** — `createGenUITools` for JSON UI mutation (get/add/delete nodes, set props, reorder) +- **System prompt** — `buildGenUISystemPrompt` for model instructions +- **Renderer** — `GenerativeUIView` passes `GEN_UI_STYLES` (overridable) to the default `GenUINode` and supports a custom node renderer + +## Why this package? + +There exists a great library for streaming interfaces: [`json-render`](https://github.com/vercel-labs/json-render). The full specification is provided in the ['Prior art'](./prior-art.md) section, but **TL;DR**: this library - `@react-native-ai/json-ui` - is the choice for small language models (e.g. parameters in the order of magnitude of 3B), which is usually the case if you are running inference locally, on-device. If you are using a cloud provider, consider `json-render` instead. + +## Requirements + +- React Native app +- Vercel AI SDK tool-calling flow (`streamText`, `generateText`, etc.) +- A model that supports tool calling + +## Installation + +```bash +bun add @react-native-ai/json-ui +``` + +## Quick Start + +```ts +import { streamText } from 'ai' +import { + buildGenUISystemPrompt, + createGenUITools, +} from '@react-native-ai/json-ui' + +const tools = createGenUITools({ + contextId: chatId, + getSpec: (id) => getSpecForChat(id), + updateSpec: (id, nextSpec) => setSpecForChat(id, nextSpec), + toolWrapper: (toolName, execute) => async (args) => { + console.log('Executing tool', toolName, args) + return execute(args) + }, +}) + +const result = streamText({ + model, + messages, + tools, + system: buildGenUISystemPrompt({ + additionalInstructions: + 'Your name is John. Keep responses short. Ask follow-up questions when UI intent is unclear.', + }), +}) +``` + +Finally, render the spec set by `updateSpec` in your app with `GenerativeUIView` (see [Generative UI View](./view.md)): + +```tsx + +``` + +For a full usage example, see [this file](https://github.com/callstackincubator/ai/tree/main/generative-ui/apps/expo-example/src/store/chatStore.ts). + +## Registries + +The package exports these registry constants (used by tools and the default renderer): + +- `GEN_UI_NODE_NAMES` — canonical node type names (Text, Paragraph, Label, Heading, Button, TextInput) +- `GEN_UI_NODE_HINTS` — short descriptions for each node type (for prompts) +- `GEN_UI_NODE_NAMES_THAT_SUPPORT_CHILDREN` — node types that can have children +- `GEN_UI_STYLES` — zod schemas for style props (flex, padding, gap, backgroundColor, color, etc.) +- `GEN_UI_STYLE_HINTS` — metadata for style keys (for prompt text) + +## Notes + +- Style validation is schema-based (zod) via `GEN_UI_STYLES`. +- The package is intentionally minimal and tuned for small-model tool loops. diff --git a/website/src/docs/json-ui/prior-art.md b/website/src/docs/json-ui/prior-art.md new file mode 100644 index 00000000..f5af2d96 --- /dev/null +++ b/website/src/docs/json-ui/prior-art.md @@ -0,0 +1,18 @@ +# Prior Art + +[json-render](https://github.com/vercel-labs/json-render) is an alternative that predates this library. It provides React and React Native integration and works with the [AI SDK](https://github.com/vercel/ai). Its design: + +- The LLM **streams** the UI in a predefined format; a stream parser consumes the output. +- A **long system prompt** describes that format and includes examples. + +That works well for **large language models** (e.g. cloud APIs). For **on-device, small models** (e.g. Apple Foundation Models with limited context), you run into: + +1. **Context size** — Small models often have 4K-token windows (such as Apple Foundation having a 4096 token limit). A long system prompt plus conversation leaves little room; you end up summarizing or truncating, if you even fit into the window at all. +2. **Task complexity** — json-render supports rich actions and state. For small models (e.g. 3B parameters), generating a correct static UI is already hard; a simpler, tool-based flow is more reliable. + +## How this package differs + +- **Tool calling instead of streaming UI** — The model emits small JSON payloads by calling tools (add node, set props, etc.). Each step is small and easier for the model to get right. +- **Narrower feature set** — Focus on static UI building first, so smaller models can complete the task. More features will be added later. + +Choose **@react-native-ai/json-ui** when you run small models on-device; consider **json-render** when using cloud providers or larger models. diff --git a/website/src/docs/json-ui/tools.md b/website/src/docs/json-ui/tools.md new file mode 100644 index 00000000..79923cfb --- /dev/null +++ b/website/src/docs/json-ui/tools.md @@ -0,0 +1,55 @@ +# Tools + +`createGenUITools` returns a set of tool definitions the model can call to read and mutate a JSON UI spec. + +## createGenUITools(options) + +Creates tools that operate on a spec with shape `{ root, elements }`. Options: + +| Option | Required | Description | +| -------------------------------- | -------- | ----------------------------------------------------------------------- | +| `contextId` | Yes | String key for the current conversation/context | +| `getSpec(contextId)` | Yes | Return current spec or `null` | +| `updateSpec(contextId, spec)` | Yes | Persist the updated spec | +| `toolWrapper(toolName, execute)` | Yes | Wrapper for every tool execution (logging, error handling, telemetry) | +| `createId` | No | Id factory for new nodes (default: generates `UI-{timestamp}-{random}`) | +| `rootId` | No | Root node id (default: `"root"`) | +| `nodeHints` | No | Override component registry used in prompts | +| `nodeNamesThatSupportChildren` | No | Override list of node types that can have children | + +### Tool list + +- **getUIRootNode** — Return the root node id and element +- **getUINode** — Return a node by id +- **getUILayout** — Return layout (root + children structure) +- **getAvailableUINodes** — List available node types and hints +- **setUINodeProps** — Set props on a node (`{ id, props, replace? }`; default merge, use `replace: true` to replace all) +- **deleteUINode** — Remove a node +- **addUINode** — Add a node (`{ parentId?, type, props? }`; omit `parentId` to use root) +- **reorderUINodes** — Move a sibling by index (`nodeId`, `offset`; negative = earlier, positive = later; clamped to valid range) + +All mutations are serialized so concurrent tool calls do not interleave writes. + +## buildGenUISystemPrompt(options?) + +Builds the reusable system instructions for JSON UI tooling. Use as the `system` option for `streamText` / `generateText`. + +Options: + +| Option | Default | Description | +| ------------------------------------ | ------- | ---------------------------------------------------------------- | +| `additionalInstructions` | — | App-specific text appended to the prompt | +| `requireLayoutReadBeforeAddingNodes` | `true` | Whether to instruct the model to read layout before adding nodes | +| `styleHints` | — | Override style metadata used in the prompt text | + +Example: + +```ts +system: buildGenUISystemPrompt({ + additionalInstructions: + 'Your name is John. Prefer short labels. Use Button for primary actions only.', + styleHints: { + borderRadius: { type: 'number', description: 'Border radius in px.' }, + }, +}) +``` diff --git a/website/src/docs/json-ui/view.md b/website/src/docs/json-ui/view.md new file mode 100644 index 00000000..7d48e881 --- /dev/null +++ b/website/src/docs/json-ui/view.md @@ -0,0 +1,99 @@ +# Generative UI View + +`GenerativeUIView` renders a JSON UI spec in React Native. The default node renderer (`GenUINode`) receives style validators from the view; you can override styles or supply a custom renderer. + +For a full usage example, see [this file](https://github.com/callstackincubator/ai/tree/main/generative-ui/apps/expo-example/src/screens/ChatScreen/ChatMessages.tsx). + +## Basic usage + +```tsx +import { GenerativeUIView } from '@react-native-ai/json-ui' +; +``` + +## Props + +| Prop | Type | Description | +| --------------------- | -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| `spec` | `{ root, elements }` or `null` / `undefined` | The UI spec to render | +| `loading` | `boolean` | Optional; shows loading placeholder when truthy and spec is empty | +| `showCollapsibleJSON` | `boolean` | Optional; shows an expandable JSON debug panel | +| `styles` | object | Optional; merged with default `GEN_UI_STYLES` and passed to `GenUINode` | +| `GenUINodeComponent` | component | Optional; custom component to render the tree; receives `{ nodeId, elements, styles }`; delegate to default `GenUINode` for nodes you don’t handle | + +## Styles from GenerativeUIView + +The default `GenUINode` does not import `GEN_UI_STYLES` itself. `GenerativeUIView` merges the registry default with any `styles` prop and passes the result down as the `styles` prop to `GenUINode`. All style validation is driven by what the view provides, so you can pass custom validators: + +```tsx +import { z } from 'zod' +import { GenerativeUIView, GEN_UI_STYLES } from '@react-native-ai/json-ui' +; +``` + +## Custom GenUINode + +You can supply your own node renderer and reuse the library’s style/prop parsing via `parseGenUIElementProps`. For one type render a custom component; for everything else use the default `GenUINode`. Pass `GenUINodeComponent` through so custom types work at any depth. + +Example: custom `Badge` type, default renderer for the rest. + +```tsx +import { View, Text, StyleSheet } from 'react-native' +import { + GenerativeUIView, + GenUINode, + parseGenUIElementProps, + type GenUINodeProps, +} from '@react-native-ai/json-ui' + +const BADGE_TYPE = 'Badge' + +function CustomGenUINode({ + nodeId, + elements, + styles, +}: GenUINodeProps) { + const element = elements[nodeId] + if (!element) return null + if (element.type === BADGE_TYPE) { + const { baseStyle, text } = parseGenUIElementProps(element, styles, { + nodeId, + type: BADGE_TYPE, + }) + return ( + + {text ?? ''} + + ) + } + return ( + + ) +} + +const customStyles = StyleSheet.create({ + badge: { + alignSelf: 'flex-start', + paddingHorizontal: 8, + paddingVertical: 4, + borderRadius: 12, + }, + badgeText: { fontSize: 12, color: '#fff' }, +}) + +// Use the custom renderer; styles still come from the view (default or overridden). + +``` + +`parseGenUIElementProps(element, styleValidators, options?)` returns `{ baseStyle, text, label, props }` and runs the same validation and prop parsing as the default renderer. Use it for custom node types that should respect the same style schema.