fix: use file-based lock (fcntl.flock) for cross-process tmux synchronization

The threading.Lock() only works within a single process. When uvicorn
spawns multiple workers or when there are multiple Python processes,
each has its own lock.

This change uses fcntl.flock() on a shared lock file, which works
across processes since they all share the same filesystem.

Author: openhands-agent

openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py

@@ -1,5 +1,7 @@
 """Tmux-based terminal backend implementation."""
 
+import fcntl
+import os
 import threading
 import time
 import uuid
@@ -16,12 +18,14 @@
 
 logger = get_logger(__name__)
 
-# Global lock to serialize tmux session creations
-# This prevents race conditions in libtmux when many agents start simultaneously
-_TMUX_SESSION_LOCK = threading.Lock()
+# File-based lock for tmux session creation
+# Uses fcntl.flock() which works across multiple processes (e.g., uvicorn workers)
+# Also keeps a threading lock for safety within the same process
+_TMUX_LOCK_FILE = "/tmp/openhands-tmux-session.lock"
+_TMUX_THREAD_LOCK = threading.Lock()
 _LOCK_COUNTER = 0  # Debug counter to track lock acquisitions
 
-logger.info(f"tmux_terminal module loaded, lock id={id(_TMUX_SESSION_LOCK)}")
+logger.info(f"tmux_terminal module loaded, lock file={_TMUX_LOCK_FILE}")
 
 
 class TmuxTerminal(TerminalInterface):
@@ -62,46 +66,60 @@ def initialize(self) -> None:
         logger.debug(f"Initializing tmux terminal with command: {window_command}")
         session_name = f"openhands-{self.username}-{uuid.uuid4()}"
 
-        # Serialize tmux session creation to prevent race conditions
+        # Serialize tmux session creation using both file lock (cross-process)
+        # and thread lock (within process) to handle all concurrency cases
         global _LOCK_COUNTER
         thread_id = threading.current_thread().ident
-        logger.info(f"[{session_name[:20]}] Thread {thread_id} waiting for lock (counter={_LOCK_COUNTER})")
-        with _TMUX_SESSION_LOCK:
-            _LOCK_COUNTER += 1
-            my_counter = _LOCK_COUNTER
-            logger.info(f"[{session_name[:20]}] Thread {thread_id} ACQUIRED lock #{my_counter}")
-            # Retry session creation to handle race conditions in libtmux
-            # where the session is created but can't be found immediately
-            max_retries = 3
-            retry_delay = 0.5
-            last_error = None
-            for attempt in range(max_retries):
-                try:
-                    logger.info(f"[{session_name[:20]}] Attempt {attempt + 1}/{max_retries} - calling new_session")
-                    self.session = self.server.new_session(
-                        session_name=session_name,
-                        start_directory=self.work_dir,
-                        kill_session=True,
-                        x=1000,
-                        y=1000,
-                    )
-                    logger.info(f"[{session_name[:20]}] SUCCESS! Session created: {self.session.name}")
-                    break
-                except TmuxObjectDoesNotExist as e:
-                    last_error = e
-                    logger.warning(
-                        f"[{session_name[:20]}] Attempt {attempt + 1}/{max_retries} FAILED: {e}"
-                    )
-                    if attempt < max_retries - 1:
-                        logger.info(f"[{session_name[:20]}] Sleeping {retry_delay}s before retry...")
-                        time.sleep(retry_delay)
-                        retry_delay *= 2  # Exponential backoff
-            else:
-                logger.error(f"[{session_name[:20]}] All {max_retries} attempts FAILED")
-                raise RuntimeError(
-                    f"Failed to create tmux session after {max_retries} attempts"
-                ) from last_error
-            logger.info(f"[{session_name[:20]}] Thread {thread_id} RELEASING lock #{my_counter}")
+        pid = os.getpid()
+        short_name = session_name[:20]
+        
+        logger.info(f"[{short_name}] PID {pid} Thread {thread_id} waiting for locks (counter={_LOCK_COUNTER})")
+        
+        # Acquire thread lock first (fast path for same-process threads)
+        with _TMUX_THREAD_LOCK:
+            # Then acquire file lock (for cross-process synchronization)
+            lock_fd = os.open(_TMUX_LOCK_FILE, os.O_CREAT | os.O_RDWR)
+            try:
+                fcntl.flock(lock_fd, fcntl.LOCK_EX)
+                _LOCK_COUNTER += 1
+                my_counter = _LOCK_COUNTER
+                logger.info(f"[{short_name}] PID {pid} Thread {thread_id} ACQUIRED both locks #{my_counter}")
+                
+                # Retry session creation to handle race conditions in libtmux
+                max_retries = 3
+                retry_delay = 0.5
+                last_error = None
+                for attempt in range(max_retries):
+                    try:
+                        logger.info(f"[{short_name}] Attempt {attempt + 1}/{max_retries} - calling new_session")
+                        self.session = self.server.new_session(
+                            session_name=session_name,
+                            start_directory=self.work_dir,
+                            kill_session=True,
+                            x=1000,
+                            y=1000,
+                        )
+                        logger.info(f"[{short_name}] SUCCESS! Session created: {self.session.name}")
+                        break
+                    except TmuxObjectDoesNotExist as e:
+                        last_error = e
+                        logger.warning(
+                            f"[{short_name}] Attempt {attempt + 1}/{max_retries} FAILED: {e}"
+                        )
+                        if attempt < max_retries - 1:
+                            logger.info(f"[{short_name}] Sleeping {retry_delay}s before retry...")
+                            time.sleep(retry_delay)
+                            retry_delay *= 2  # Exponential backoff
+                else:
+                    logger.error(f"[{short_name}] All {max_retries} attempts FAILED")
+                    raise RuntimeError(
+                        f"Failed to create tmux session after {max_retries} attempts"
+                    ) from last_error
+                    
+                logger.info(f"[{short_name}] PID {pid} Thread {thread_id} RELEASING locks #{my_counter}")
+            finally:
+                fcntl.flock(lock_fd, fcntl.LOCK_UN)
+                os.close(lock_fd)
         for k, v in env.items():
             self.session.set_environment(k, v)