From 6370433ac37a4593a9d964e14cf41e60d0c83cb4 Mon Sep 17 00:00:00 2001
From: MaxMagician <leon.zheng2016@gmail.com>
Date: Thu, 11 Dec 2025 01:47:47 -0800
Subject: [PATCH 1/2] Integration: Add proper Ollama support with Qwen3-VL
 default

- Added Ollama provider support to LLM API layer.
- Implemented  to fix multimodal message formatting compatibility (resolves 400 Bad Request).
- Fixed  in agent retry logic.
- Fixed  in multimodal token counting.
- Updated default model configuration to  in ,  (hardcoded default), and .
- Added new models to frontend settings list.
- Updated  default model state.
---
 frontend/src/App.js                      | 63 +++++++++++-----------
 frontend/src/components/SettingsPanel.js | 16 +++++-
 mirix/agent/agent.py                     |  1 +
 mirix/agent/agent_wrapper.py             | 16 +++++-
 mirix/agent/app_constants.py             | 10 ++++
 mirix/configs/mirix.yaml                 |  6 +++
 mirix/configs/mirix_monitor.yaml         |  6 +++
 mirix/configs/mirix_ollama.yaml          |  6 +++
 mirix/llm_api/llm_api_tools.py           | 69 +++++++++++++++++++++++-
 mirix/llm_api/ollama_helpers.py          | 40 ++++++++++++++
 mirix/schemas/message.py                 |  6 ++-
 mirix/utils.py                           | 29 ++++++++--
 12 files changed, 228 insertions(+), 40 deletions(-)
 create mode 100644 mirix/configs/mirix_ollama.yaml
 create mode 100644 mirix/llm_api/ollama_helpers.py

diff --git a/frontend/src/App.js b/frontend/src/App.js
index 7273dcde..6920f06d 100644
--- a/frontend/src/App.js
+++ b/frontend/src/App.js
@@ -16,6 +16,7 @@ function App() {
   const [activeTab, setActiveTab] = useState('chat');
   const [settings, setSettings] = useState({
     model: 'gpt-4o-mini',
+    memoryModel: 'gpt-4o-mini',
     persona: 'helpful_assistant',
     timezone: 'America/New_York',
     serverUrl: 'http://localhost:47283'
@@ -57,7 +58,7 @@ function App() {
       if (response.ok) {
         const data = await response.json();
         console.log('API key status:', data);
-        
+
         if (forceOpen || (data.requires_api_key && data.missing_keys.length > 0)) {
           if (forceOpen) {
             console.log('Manual API key update requested');
@@ -88,10 +89,10 @@ function App() {
   // Refresh backend-dependent data after successful connection
   const refreshBackendData = useCallback(async () => {
     console.log('🔄 Refreshing backend-dependent data...');
-    
+
     // Check API keys after successful backend connection
     await checkApiKeys();
-    
+
     // Trigger refresh of other backend-dependent components
     // This will cause components like SettingsPanel to re-fetch their data
     setSettings(prev => ({ ...prev, lastBackendRefresh: Date.now() }));
@@ -101,7 +102,7 @@ function App() {
   const checkBackendHealth = useCallback(async () => {
     let shouldProceed = true;
     let currentVisibility = false;
-    
+
     // Check if health check is already in progress and capture current visibility
     setBackendLoading(prev => {
       if (prev.isChecking) {
@@ -133,7 +134,7 @@ function App() {
 
       if (response.ok) {
         console.log('✅ Backend is healthy - hiding loading modal');
-        
+
         setBackendLoading(prev => ({
           ...prev,
           isVisible: false,
@@ -148,7 +149,7 @@ function App() {
           console.log('🔄 Backend reconnected - refreshing data...');
           await refreshBackendData();
         }
-        
+
         return true;
       } else {
         throw new Error(`Health check failed with status: ${response.status}`);
@@ -187,15 +188,15 @@ function App() {
   useEffect(() => {
     const performInitialHealthCheck = async () => {
       // Show loading modal immediately for initial startup
-      setBackendLoading(prev => ({ 
-        ...prev, 
-        isVisible: true, 
+      setBackendLoading(prev => ({
+        ...prev,
+        isVisible: true,
         isReconnection: false // This is initial startup, not reconnection
       }));
-      
+
       // Wait a moment for the UI to update
       await new Promise(resolve => setTimeout(resolve, 500));
-      
+
       // Check backend health (will automatically refresh data if modal was visible)
       await checkBackendHealth();
     };
@@ -208,17 +209,17 @@ function App() {
     const interval = setInterval(() => {
       setBackendLoading(prev => {
         const timeSinceLastCheck = Date.now() - (prev.lastCheckTime || 0);
-        
+
         // Check more frequently when modal is visible, less frequently when not
-        const shouldCheck = prev.isVisible 
+        const shouldCheck = prev.isVisible
           ? !prev.isChecking // Every 5 seconds when modal is visible
           : timeSinceLastCheck > 30000 && !prev.isChecking; // Every 30 seconds when modal is hidden
-        
+
         if (shouldCheck) {
           console.log('🔄 Periodic health check triggered. Modal visible:', prev.isVisible);
           checkBackendHealth();
         }
-        
+
         return prev; // Don't actually update state, just check conditions
       });
     }, 5000); // Check every 5 seconds
@@ -230,7 +231,7 @@ function App() {
   useEffect(() => {
     const handleWindowFocus = async () => {
       console.log('🔍 Window focused - checking backend health silently...');
-      
+
       // Check backend health silently - only show modal if it actually fails
       const healthCheckResult = await checkBackendHealth();
       // Loading modal will be shown automatically by checkBackendHealth if it fails
@@ -239,7 +240,7 @@ function App() {
     const handleVisibilityChange = async () => {
       if (!document.hidden) {
         console.log('🔍 Document became visible - checking backend health silently...');
-        
+
         // Check backend health silently - only show modal if it actually fails
         const healthCheckResult = await checkBackendHealth();
         // Loading modal will be shown automatically by checkBackendHealth if it fails
@@ -264,7 +265,7 @@ function App() {
   const handleApiKeySubmit = async () => {
     // Refresh API key status after submission
     await checkApiKeys();
-    
+
     // If there's a pending model change, retry it now
     if (pendingModelChange.retryFunction) {
       console.log(`Retrying ${pendingModelChange.type} model change to '${pendingModelChange.model}' after API key update`);
@@ -285,7 +286,7 @@ function App() {
   useEffect(() => {
     // Listen for menu events from Electron
     const cleanupFunctions = [];
-    
+
     if (window.electronAPI) {
       const cleanupNewChat = window.electronAPI.onMenuNewChat(() => {
         setActiveTab('chat');
@@ -309,7 +310,7 @@ function App() {
       // Handle Electron window events - check backend health silently
       const cleanupWindowShow = window.electronAPI.onWindowShow(async () => {
         console.log('🔍 Electron window shown - checking backend health silently...');
-        
+
         // Check backend health silently - only show modal if it actually fails
         const healthCheckResult = await checkBackendHealth();
         // Loading modal will be shown automatically by checkBackendHealth if it fails
@@ -318,7 +319,7 @@ function App() {
 
       const cleanupAppActivate = window.electronAPI.onAppActivate(async () => {
         console.log('🔍 Electron app activated - checking backend health silently...');
-        
+
         // Check backend health silently - only show modal if it actually fails
         const healthCheckResult = await checkBackendHealth();
         // Loading modal will be shown automatically by checkBackendHealth if it fails
@@ -344,32 +345,32 @@ function App() {
     <div className="App">
       <div className="app-header">
         <div className="app-title">
-          <Logo 
-            size="small" 
-            showText={false} 
+          <Logo
+            size="small"
+            showText={false}
           />
           <span className="version">v0.1.5</span>
         </div>
         <div className="tabs">
-          <button 
+          <button
             className={`tab ${activeTab === 'chat' ? 'active' : ''}`}
             onClick={() => setActiveTab('chat')}
           >
             {t('tabs.chat')}
           </button>
-          <button 
+          <button
             className={`tab ${activeTab === 'screenshots' ? 'active' : ''}`}
             onClick={() => setActiveTab('screenshots')}
           >
             {t('tabs.screenshots')}
           </button>
-          <button 
+          <button
             className={`tab ${activeTab === 'memory' ? 'active' : ''}`}
             onClick={() => setActiveTab('memory')}
           >
             {t('tabs.memory')}
           </button>
-          <button 
+          <button
             className={`tab ${activeTab === 'settings' ? 'active' : ''}`}
             onClick={() => setActiveTab('settings')}
           >
@@ -380,7 +381,7 @@ function App() {
 
       <div className="app-content">
         {/* Keep ChatWindow always mounted to maintain streaming state */}
-        <div style={{ 
+        <div style={{
           display: activeTab === 'chat' ? 'flex' : 'none',
           flexDirection: 'column',
           height: '100%'
@@ -401,8 +402,8 @@ function App() {
         </div>
         {/* Keep ScreenshotMonitor always mounted to maintain monitoring state */}
         <div style={{ display: activeTab === 'screenshots' ? 'block' : 'none' }}>
-          <ScreenshotMonitor 
-            settings={settings} 
+          <ScreenshotMonitor
+            settings={settings}
             onMonitoringStatusChange={setIsScreenMonitoring}
           />
         </div>
diff --git a/frontend/src/components/SettingsPanel.js b/frontend/src/components/SettingsPanel.js
index a02f3555..0e421200 100644
--- a/frontend/src/components/SettingsPanel.js
+++ b/frontend/src/components/SettingsPanel.js
@@ -984,7 +984,14 @@ const SettingsPanel = ({ settings, onSettingsChange, onApiKeyCheck, onApiKeyRequ
     'gemini-2.5-flash',
     'gemini-2.5-flash-lite',
     'gemini-1.5-pro',
-    'gemini-2.0-flash-lite'
+    'gemini-2.0-flash-lite',
+    'llama3.2',
+    'mistral',
+    'gemma2',
+    'qwen2.5-coder-32b',
+    'deepseek-r1-distill-llama-70b',
+    'deepseek-v3.1:671b-cloud',
+    'qwen3-vl:235b-cloud'
   ];
 
   // Combine base models with custom models
@@ -999,6 +1006,13 @@ const SettingsPanel = ({ settings, onSettingsChange, onApiKeyCheck, onApiKeyRequ
     'gpt-4o',
     'gpt-4.1-mini',
     'gpt-4.1',
+    'llama3.2',
+    'mistral',
+    'gemma2',
+    'qwen2.5-coder-32b',
+    'deepseek-r1-distill-llama-70b',
+    'deepseek-v3.1:671b-cloud',
+    'qwen3-vl:235b-cloud'
   ];
 
   // Combine base memory models with custom models
diff --git a/mirix/agent/agent.py b/mirix/agent/agent.py
index 0d2a78c6..332ba968 100644
--- a/mirix/agent/agent.py
+++ b/mirix/agent/agent.py
@@ -725,6 +725,7 @@ def _get_ai_reply(
                         f"Attempt {attempt} failed: {he}. Retrying in {delay} seconds..."
                     )
                     time.sleep(delay)
+                    continue
 
             except Exception as e:
                 log_telemetry(
diff --git a/mirix/agent/agent_wrapper.py b/mirix/agent/agent_wrapper.py
index 1f6186dc..a3f2933a 100644
--- a/mirix/agent/agent_wrapper.py
+++ b/mirix/agent/agent_wrapper.py
@@ -20,6 +20,7 @@
     GEMINI_MODELS,
     MAXIMUM_NUM_IMAGES_IN_CLOUD,
     OPENAI_MODELS,
+    OLLAMA_MODELS,
     TEMPORARY_MESSAGE_LIMIT,
 )
 
@@ -882,6 +883,8 @@ def _determine_model_provider(
             return "anthropic"
         elif model_name in OPENAI_MODELS:
             return "openai"
+        elif model_name in OLLAMA_MODELS:
+            return "ollama"
         else:
             raise ValueError(f"Invalid model provider: {model_name}")
 
@@ -962,6 +965,15 @@ def _create_llm_config_for_provider(
                     model_wrapper=None,
                     context_window=128000,
                 )
+
+        elif provider == "ollama":
+            llm_config = LLMConfig(
+                model=model_name,
+                model_endpoint_type="ollama",
+                model_endpoint=self.agent_config.get("ollama_base_url") or model_settings.ollama_base_url or "http://localhost:11434/v1",
+                model_wrapper=None,
+                context_window=128000,
+            )
         else:
             # Custom provider - use custom config or fallback to OpenAI-compatible
             if custom_agent_config is not None:
@@ -1051,7 +1063,7 @@ def set_memory_model(self, new_model, custom_agent_config: dict = None):
         """Set the model specifically for memory management operations"""
 
         # Define allowed memory models
-        ALLOWED_MEMORY_MODELS = GEMINI_MODELS + OPENAI_MODELS
+        ALLOWED_MEMORY_MODELS = GEMINI_MODELS + OPENAI_MODELS + OLLAMA_MODELS
 
         # Determine the effective provider
         provider = self._determine_model_provider(new_model, custom_agent_config)
@@ -1105,6 +1117,8 @@ def set_memory_model(self, new_model, custom_agent_config: dict = None):
             required_keys = ["GEMINI_API_KEY"]
         elif new_model in OPENAI_MODELS:
             required_keys = ["OPENAI_API_KEY"]
+        elif new_model in OLLAMA_MODELS:
+             required_keys = [] # Ollama usually doesn't require an API key
 
         return {
             "success": True,
diff --git a/mirix/agent/app_constants.py b/mirix/agent/app_constants.py
index ae32bfcd..7b8c92a2 100644
--- a/mirix/agent/app_constants.py
+++ b/mirix/agent/app_constants.py
@@ -18,6 +18,16 @@
     "gpt-5",
 ]
 
+OLLAMA_MODELS = [
+    "llama3.2",
+    "mistral",
+    "gemma2",
+    "qwen2.5-coder-32b",
+    "deepseek-r1-distill-llama-70b",
+    "deepseek-v3.1:671b-cloud",
+    "qwen3-vl:235b-cloud",
+]
+
 STUCK_TIMEOUT = 10
 RUNNING_TIMEOUT = 30
 TOTAL_TIMEOUT = 60
diff --git a/mirix/configs/mirix.yaml b/mirix/configs/mirix.yaml
index 3c36060e..ba492c8c 100644
--- a/mirix/configs/mirix.yaml
+++ b/mirix/configs/mirix.yaml
@@ -1,2 +1,8 @@
 agent_name: mirix
 model_name: gemini-2.5-flash-lite
+# Ollama support - uncomment to use Ollama instead
+# model_name: qwen3-vl:235b-cloud
+# model_endpoint_type: ollama
+# model_endpoint: http://localhost:11434/v1
+# context_window: 16384
+# put_inner_thoughts_in_kwargs: true
diff --git a/mirix/configs/mirix_monitor.yaml b/mirix/configs/mirix_monitor.yaml
index 14be07d4..6e4cbad4 100644
--- a/mirix/configs/mirix_monitor.yaml
+++ b/mirix/configs/mirix_monitor.yaml
@@ -1,3 +1,9 @@
 agent_name: mirix
 model_name: gemini-2.0-flash
+# Ollama support - uncomment to use Ollama instead
+# model_name: qwen3-vl:235b-cloud
+# model_endpoint_type: ollama
+# model_endpoint: http://localhost:11434/v1
+# context_window: 16384
+# put_inner_thoughts_in_kwargs: true
 is_screen_monitor: true
\ No newline at end of file
diff --git a/mirix/configs/mirix_ollama.yaml b/mirix/configs/mirix_ollama.yaml
new file mode 100644
index 00000000..f50617de
--- /dev/null
+++ b/mirix/configs/mirix_ollama.yaml
@@ -0,0 +1,6 @@
+agent_name: mirix
+model_name: qwen3-vl:235b-cloud
+model_endpoint_type: ollama
+model_endpoint: http://localhost:11434/v1
+context_window: 16384
+put_inner_thoughts_in_kwargs: true
diff --git a/mirix/llm_api/llm_api_tools.py b/mirix/llm_api/llm_api_tools.py
index 57f3d238..0fde6289 100755
--- a/mirix/llm_api/llm_api_tools.py
+++ b/mirix/llm_api/llm_api_tools.py
@@ -20,6 +20,7 @@
     convert_tools_to_google_ai_format,
     google_ai_chat_completions_request,
 )
+from mirix.llm_api.ollama_helpers import _preprocess_ollama_messages
 from mirix.llm_api.helpers import (
     add_inner_thoughts_to_functions,
     unpack_all_inner_thoughts_from_kwargs,
@@ -37,7 +38,7 @@
 )
 from mirix.schemas.openai.chat_completion_response import ChatCompletionResponse
 from mirix.settings import ModelSettings
-from mirix.utils import num_tokens_from_functions, num_tokens_from_messages
+from mirix.utils import num_tokens_from_functions, num_tokens_from_messages, smart_urljoin
 
 LLM_API_PROVIDER_OPTIONS = [
     "openai",
@@ -47,6 +48,7 @@
     "cohere",
     "local",
     "groq",
+    "ollama",
 ]
 
 
@@ -135,6 +137,9 @@ def create(
     image_uris: Optional[List[str]] = None,  # TODO: inside messages
     extra_messages: Optional[List[Message]] = None,
     get_input_data_for_debugging: bool = False,
+    user_id: Optional[str] = None,
+    put_inner_thoughts_first: bool = False,
+    name: Optional[str] = None,
 ) -> ChatCompletionResponse:
     """Return response to chat completion with backoff"""
     from mirix.utils import printd
@@ -559,6 +564,68 @@ def create(
             ),
         )
 
+    # ollama
+    elif llm_config.model_endpoint_type == "ollama":
+        if stream:
+            raise NotImplementedError(
+                f"Streaming not yet implemented for {llm_config.model_endpoint_type}"
+            )
+
+        if not model_settings.ollama_base_url and not llm_config.model_endpoint:
+            # Default to localhost if not set
+            llm_config.model_endpoint = "http://localhost:11434/v1"
+        elif model_settings.ollama_base_url and not llm_config.model_endpoint:
+            llm_config.model_endpoint = model_settings.ollama_base_url
+
+        # Ensure endpoint ends with /v1
+        if not llm_config.model_endpoint.endswith("/v1"):
+            llm_config.model_endpoint = smart_urljoin(llm_config.model_endpoint, "v1")
+
+        if llm_config.put_inner_thoughts_in_kwargs:
+            functions = add_inner_thoughts_to_functions(
+                functions=functions,
+                inner_thoughts_key=INNER_THOUGHTS_KWARG,
+                inner_thoughts_description=INNER_THOUGHTS_KWARG_DESCRIPTION,
+            )
+
+        tools = (
+            [{"type": "function", "function": f} for f in functions]
+            if functions is not None
+            else None
+        )
+
+        # Ollama is OpenAI compatible
+        data = ChatCompletionRequest(
+            model=llm_config.model,
+            messages=_preprocess_ollama_messages(
+                messages,
+                put_inner_thoughts_in_kwargs=llm_config.put_inner_thoughts_in_kwargs
+            ),
+            tools=tools,
+            tool_choice=function_call,
+            user=user_id if user_id else "user",
+        )
+
+        # Ollama typically doesn't need an API key, but we pass a dummy one if None
+        api_key = "ollama"
+
+        response = openai_chat_completions_request(
+            url=llm_config.model_endpoint,
+            api_key=api_key,
+            chat_completion_request=data,
+            get_input_data_for_debugging=get_input_data_for_debugging,
+        )
+
+        if get_input_data_for_debugging:
+            return response
+
+        if llm_config.put_inner_thoughts_in_kwargs:
+            response = unpack_all_inner_thoughts_from_kwargs(
+                response=response, inner_thoughts_key=INNER_THOUGHTS_KWARG
+            )
+
+        return response
+
     # local model
     else:
         if stream:
diff --git a/mirix/llm_api/ollama_helpers.py b/mirix/llm_api/ollama_helpers.py
new file mode 100644
index 00000000..528f3e2a
--- /dev/null
+++ b/mirix/llm_api/ollama_helpers.py
@@ -0,0 +1,40 @@
+
+def _preprocess_ollama_messages(messages, put_inner_thoughts_in_kwargs=False):
+    processed_messages = []
+    for m in messages:
+        msg_dict = m.to_openai_dict(
+            put_inner_thoughts_in_kwargs=put_inner_thoughts_in_kwargs
+        )
+        # Fix ImageContent format for OpenAI/Ollama compatibility
+        if msg_dict.get("content") and isinstance(msg_dict["content"], list):
+            new_content = []
+            for item in msg_dict["content"]:
+                if isinstance(item, dict):
+                    # Check for invalid image_id format and convert to valid image_url
+                    if item.get("type") == "image_url" and "image_id" in item:
+                        # Construct a valid OpenAI image_url object
+                        # Since we don't have easy access to the image file here to base64 encode it,
+                        # and passing a file path usually doesn't work for remote APIs (unless Ollama runs locally),
+                        # we will try to provide a text description if we can't properly link it,
+                        # OR if we assume Ollama is local, maybe we can pass the path if we had it.
+                        # But we only have image_id.
+                        # However, for now, to prevent 400 errors, we must fix the structure.
+                        # We will convert it to a text "placeholder" to avoid crashing until we can implement full image resolution.
+                        # Using a text placeholder is safer than sending invalid JSON that crashes the request.
+                        new_content.append({
+                            "type": "text", 
+                            "text": f"[Image ID: {item['image_id']}] (Image skipped due to resolution limit)"
+                        })
+                    elif item.get("type") == "image_url" and "image_url" not in item:
+                         # Fallback for any other malformed image_url
+                        new_content.append({
+                            "type": "text", 
+                            "text": "[Image: malformed content]"
+                        })
+                    else:
+                        new_content.append(item)
+                else:
+                    new_content.append(item)
+            msg_dict["content"] = new_content
+        processed_messages.append(msg_dict)
+    return processed_messages
diff --git a/mirix/schemas/message.py b/mirix/schemas/message.py
index ad322364..c8956c75 100644
--- a/mirix/schemas/message.py
+++ b/mirix/schemas/message.py
@@ -801,12 +801,14 @@ def to_openai_dict(
                         }
                     )
                 elif isinstance(content_part, CloudFileContent):
+                     # Fallback for OpenAI/Ollama which don't support Google Cloud URIs
                     content.append(
                         {
-                            "type": content_part.type,
-                            "cloud_file_uri": content_part.cloud_file_uri,
+                            "type": "text",
+                            "text": f"[Image File: {content_part.cloud_file_uri}]",
                         }
                     )
+                    text_content_count += 1
                 else:
                     raise ValueError(f"Invalid content type: {content_part.type}")
 
diff --git a/mirix/utils.py b/mirix/utils.py
index 20e431d7..29bbde2f 100755
--- a/mirix/utils.py
+++ b/mirix/utils.py
@@ -1441,12 +1441,33 @@ def num_tokens_from_messages(messages: List[dict], model: str = "gpt-4") -> int:
                         warnings.warn(
                             f"Message has null value: {key} with value: {value} - message={message}"
                         )
+                    elif isinstance(value, list) and key == "content":
+                        # Handle multimodal content (list of dicts)
+                        for item in value:
+                            if isinstance(item, dict):
+                                if item.get("type") == "text":
+                                    num_tokens += len(encoding.encode(item.get("text", "")))
+                                elif item.get("type") == "image_url":
+                                    # Approximate tokens for an image
+                                    # OpenAI pricing/sizing varies, but a safe estimate or fixed cost is often used
+                                    # High detail mode: 85 tokens + 170 * tiles. Low detail: 85 tokens.
+                                    # For now, let's assume a fixed cost or simple estimate to avoid crashing.
+                                    # 85 tokens is a reasonable minimum.
+                                    num_tokens += 85 
                     else:
                         if not isinstance(value, str):
-                            raise ValueError(
-                                f"Message has non-string value: {key} with value: {value} - message={message}"
-                            )
-                        num_tokens += len(encoding.encode(value))
+                            # Try to stringify if not string, or skip if safe?
+                            # For consistency with previous logic, we should probably warn or try to convert.
+                            # But strictly, let's keep the error for unknown non-string types unless it's the list case we just handled.
+                             try:
+                                value = str(value)
+                                num_tokens += len(encoding.encode(value))
+                             except:
+                                raise ValueError(
+                                    f"Message has non-string value: {key} with value: {value} - message={message}"
+                                )
+                        else:
+                            num_tokens += len(encoding.encode(value))
 
                 if key == "name":
                     num_tokens += tokens_per_name

From 1cc40cf72c784c07bbde435ff81d076f9eb43220 Mon Sep 17 00:00:00 2001
From: MaxMagician <leon.zheng2016@gmail.com>
Date: Thu, 11 Dec 2025 18:59:07 -0800
Subject: [PATCH 2/2] feat: Implement real image support for Ollama vision
 models

- Convert image_id to base64-encoded images for Ollama
- Use FileManager to retrieve image file paths from database
- Support both source_url and local file paths
- Add proper error handling and fallbacks
- Enable vision models like qwen3-vl to actually see and analyze images

Tested with qwen3-vl:235b-cloud - successfully identifies shapes, colors, and text in images.
---
 mirix/llm_api/ollama_helpers.py | 77 +++++++++++++++++++++++++--------
 1 file changed, 60 insertions(+), 17 deletions(-)

diff --git a/mirix/llm_api/ollama_helpers.py b/mirix/llm_api/ollama_helpers.py
index 528f3e2a..0fe79f21 100644
--- a/mirix/llm_api/ollama_helpers.py
+++ b/mirix/llm_api/ollama_helpers.py
@@ -1,40 +1,83 @@
+import base64
+import mimetypes
+from typing import List
+
 
 def _preprocess_ollama_messages(messages, put_inner_thoughts_in_kwargs=False):
+    """
+    Preprocess messages for Ollama compatibility.
+    Converts image_id references to base64-encoded images that Ollama can understand.
+    """
+    from mirix.services.file_manager import FileManager
+    
+    file_manager = FileManager()
     processed_messages = []
+    
     for m in messages:
         msg_dict = m.to_openai_dict(
             put_inner_thoughts_in_kwargs=put_inner_thoughts_in_kwargs
         )
-        # Fix ImageContent format for OpenAI/Ollama compatibility
+        
+        # Fix ImageContent format for Ollama compatibility
         if msg_dict.get("content") and isinstance(msg_dict["content"], list):
             new_content = []
             for item in msg_dict["content"]:
                 if isinstance(item, dict):
-                    # Check for invalid image_id format and convert to valid image_url
+                    # Check for image_url with image_id (needs conversion to base64)
                     if item.get("type") == "image_url" and "image_id" in item:
-                        # Construct a valid OpenAI image_url object
-                        # Since we don't have easy access to the image file here to base64 encode it,
-                        # and passing a file path usually doesn't work for remote APIs (unless Ollama runs locally),
-                        # we will try to provide a text description if we can't properly link it,
-                        # OR if we assume Ollama is local, maybe we can pass the path if we had it.
-                        # But we only have image_id.
-                        # However, for now, to prevent 400 errors, we must fix the structure.
-                        # We will convert it to a text "placeholder" to avoid crashing until we can implement full image resolution.
-                        # Using a text placeholder is safer than sending invalid JSON that crashes the request.
-                        new_content.append({
-                            "type": "text", 
-                            "text": f"[Image ID: {item['image_id']}] (Image skipped due to resolution limit)"
-                        })
+                        try:
+                            # Get file metadata from database
+                            file = file_manager.get_file_metadata_by_id(item["image_id"])
+                            
+                            if file.source_url is not None:
+                                # Use URL directly if available
+                                new_content.append({
+                                    "type": "image_url",
+                                    "image_url": {
+                                        "url": file.source_url
+                                    }
+                                })
+                            elif file.file_path is not None:
+                                # Convert local file to base64
+                                mime_type, _ = mimetypes.guess_type(file.file_path)
+                                if mime_type is None or not mime_type.startswith("image/"):
+                                    mime_type = "image/jpeg"  # Default fallback
+                                
+                                with open(file.file_path, "rb") as img_file:
+                                    base64_data = base64.b64encode(img_file.read()).decode("utf-8")
+                                    new_content.append({
+                                        "type": "image_url",
+                                        "image_url": {
+                                            "url": f"data:{mime_type};base64,{base64_data}"
+                                        }
+                                    })
+                            else:
+                                # Fallback to text placeholder if no file path available
+                                new_content.append({
+                                    "type": "text",
+                                    "text": f"[Image ID: {item['image_id']}] (Image file not found)"
+                                })
+                        except Exception as e:
+                            # If we can't load the image, use a text placeholder
+                            new_content.append({
+                                "type": "text",
+                                "text": f"[Image ID: {item['image_id']}] (Error loading image: {str(e)})"
+                            })
+                    
                     elif item.get("type") == "image_url" and "image_url" not in item:
-                         # Fallback for any other malformed image_url
+                        # Fallback for any other malformed image_url
                         new_content.append({
-                            "type": "text", 
+                            "type": "text",
                             "text": "[Image: malformed content]"
                         })
                     else:
+                        # Keep other content types as-is
                         new_content.append(item)
                 else:
                     new_content.append(item)
+            
             msg_dict["content"] = new_content
+        
         processed_messages.append(msg_dict)
+    
     return processed_messages