Changes venv creation path for superset to writeable location. Fix for MacOS venv creation permission denied.

Author: NicoLongfield

go_server/main

@@ -1,8 +1,10 @@
 import json
 import subprocess
 import sys
+import os
 import time
 from pathlib import Path
+import venv
 
 SUPERSET_PACKAGES = [
     "apache-superset==4.1.1",
@@ -12,14 +14,18 @@
 ]
 
 class SupersetEnvManager:
-    def __init__(self, python_path):
+    def __init__(self, python_path, app_data_path):
         self.python_path = python_path
         self.env_path = None
+        
+        # Using app_data_path to define environment path (defined by userData https://www.electronjs.org/docs/latest/api/app#appgetapppath)
+        # This is a writeable path where we can create our virtual environment (Fix for macOS venv creation issues)
+        base_dir = Path(app_data_path)
         if sys.platform == "win32":
-            self.env_path = Path(python_path).parent / "superset_env/Scripts/python.exe"
+            self.env_path = base_dir / "superset_env/Scripts/python.exe"
         else:
-            self.env_path = python_path.replace("bin", "bin/superset_env/bin")
-    
+            self.env_path = base_dir / "superset_env/bin/python"
+
     def check_env_exists(self):
         """Check if the virtual environment exists"""
         if sys.platform == "win32":
@@ -34,17 +40,38 @@ def create_env(self):
             env_name = str(self.env_path)[:env_name+len("superset_env")]
         else:
             return False
+        
+        print(f"Creating virtual environment at: {env_name}")
+
+        try:
+            # Create venv without pip first to avoid crash due to missing lib on macOS standalone builds
+            venv.create(env_name, with_pip=False, clear=True)
+            
+            # Fix for macOS standalone python: symlink libpython
+            if sys.platform == "darwin":
+                base_python_lib = Path(self.python_path).parent.parent / "lib"
+                venv_lib = Path(env_name) / "lib"
+                
+                # Find libpython dylib
+                lib_files = list(base_python_lib.glob("libpython*.dylib"))
+                if lib_files:
+                    target_lib = lib_files[0]
+                    link_name = venv_lib / target_lib.name
+                    if not link_name.exists():
+                        try:
+                            link_name.symlink_to(target_lib)
+                            print(f"Symlinked {target_lib} to {link_name}")
+                        except Exception as e:
+                            print(f"Failed to symlink libpython: {e}")
+
+            # Now install pip
+            print("Installing pip...")
+            subprocess.run([str(self.env_path), "-m", "ensurepip"], check=True)
 
-        # Create virtual environment
-        process = subprocess.Popen([
-            self.python_path, "-m", "venv", env_name
-        ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-        process.wait()
-        if process.returncode == 0:
             print(f"Environment created at: {self.env_path}")
             return True
-        else:
-            print(f"Error creating environment: {process.stderr}")
+        except Exception as e:
+            print(f"Error creating environment: {e}")
             return False
 
     def check_requirements(self):

pythonCode/modules/superset/SupersetEnvManager.py

@@ -34,14 +34,25 @@ def _custom_process(self, json_config: dict) -> dict:
 
         # Map settings
         python_path = json_config["pythonPath"]
+        app_data_path = json_config.get("appDataPath")
+        
+        if not app_data_path:
+            # Fallback to default locations if not provided (for backward compatibility or testing)
+            if sys.platform == "win32":
+                app_data_path = str(Path(os.getenv('APPDATA')) / "medomics-platform")
+            elif sys.platform == "darwin":
+                app_data_path = str(Path.home() / "Library/Application Support/medomics-platform")
+            else:
+                app_data_path = str(Path.home() / ".config/medomics-platform")
+
         username = json_config["username"]
         firstname = json_config["firstname"]
         lastname = json_config["lastname"]
         email = json_config["email"]
         password = json_config["password"]
 
         # Set up Superset
-        output = self.create_user(python_path, username, firstname, lastname, email, password)
+        output = self.create_user(python_path, app_data_path, username, firstname, lastname, email, password)
 
         return output
 
@@ -59,12 +70,13 @@ def run_command(self, command, env=None, capture_output=True, timeout=None):
 
         return {}
 
-    def create_user(self, python_path: str, username: str, firstname: str, lastname: str, email: str, password: str):
+    def create_user(self, python_path: str, app_data_path: str, username: str, firstname: str, lastname: str, email: str, password: str):
         """
         Creates a new Superset user.
 
         Args:
             python_path: The path to the Python installation.
+            app_data_path: The path to the application data directory.
             username: The username of the new user.
             firstname: The first name of the new user.
             lastname: The last name of the new user.
@@ -76,7 +88,7 @@ def create_user(self, python_path: str, username: str, firstname: str, lastname:
         """
         # Prepare environment variables
         env = os.environ.copy()
-        manager = SupersetEnvManager(python_path)
+        manager = SupersetEnvManager(python_path, app_data_path)
         path_superset = manager.get_superset_path()
         env["FLASK_APP"] = "superset"
 
@@ -90,7 +102,7 @@ def create_user(self, python_path: str, username: str, firstname: str, lastname:
             "password": password,
         }
         output = self.run_command(
-            f"{path_superset} fab create-admin "
+            f'"{path_superset}" fab create-admin '
             f"--username {admin_user['username']} "
             f"--firstname {admin_user['firstname']} "
             f"--lastname {admin_user['lastname']} "

pythonCode/modules/superset/create_user.py

@@ -28,14 +28,24 @@ def __init__(self, json_params: dict, _id: str = None):
 
     def _custom_process(self, json_config: dict) -> dict:
         """
-            This function predicts from a model, a dataset, and a new dataset
+            This function is called to execute the custom process which sets up Superset.
         """
         # Map settings
         port = json_config["port"]
         python_path = json_config["pythonPath"]
+        app_data_path = json_config.get("appDataPath")
+        
+        if not app_data_path:
+            # Fallback to default locations if not provided (for backward compatibility or testing)
+            if sys.platform == "win32":
+                app_data_path = str(Path(os.getenv('APPDATA')) / "medomics-platform")
+            elif sys.platform == "darwin":
+                app_data_path = str(Path.home() / "Library/Application Support/medomics-platform")
+            else:
+                app_data_path = str(Path.home() / ".config/medomics-platform")
 
         # Set up Superset
-        result = self.setup_superset(port, python_path)
+        result = self.setup_superset(port, python_path, app_data_path)
 
         return result
 
@@ -53,13 +63,14 @@ def run_command(self, command, env=None, capture_output=True, timeout=None):
             return {"error": f"Error while running command: {command}. Full error log:" + e.stderr}
 
 
-    def setup_superset(self, port, python_path):
+    def setup_superset(self, port, python_path, app_data_path):
         """
         Set up Superset with the provided settings.
         
         Args:
             scripts_path (path): The path to the Python scripts directory.
             port (path): The port on which to run Superset.
+            app_data_path (path): The path to the application data directory.
             
         Returns:
             A dictionary containing the error message, if any.
@@ -70,7 +81,7 @@ def setup_superset(self, port, python_path):
 
         # Check if the virtual environment exists
         self.set_progress(now=progress, label="Checking the Superset virtual environment...")
-        manager = SupersetEnvManager(python_path)
+        manager = SupersetEnvManager(python_path, app_data_path)
         if not manager.check_env_exists():
             print("Creating Superset virtual environment...")
             self.set_progress(now=self._progress["now"]+step, label="Creating Superset virtual environment...")
@@ -147,7 +158,7 @@ def setup_superset(self, port, python_path):
         # Initialize the database
         print("Initializing the Superset database...")
         self.set_progress(now=self._progress["now"]+step, label="Initializing the Superset database...")
-        result = self.run_command(f"{superset_path} db upgrade", env)
+        result = self.run_command(f'"{superset_path}" db upgrade', env)
         if "error" in result:
             return result
 
@@ -162,7 +173,7 @@ def setup_superset(self, port, python_path):
             "password": "admin",
         }
         result = self.run_command(
-            f"{superset_path} fab create-admin "
+            f'"{superset_path}" fab create-admin '
             f"--username {admin_user['username']} "
             f"--firstname {admin_user['firstname']} "
             f"--lastname {admin_user['lastname']} "
@@ -176,16 +187,17 @@ def setup_superset(self, port, python_path):
         # Initialize Superset
         print("Initializing Superset...")
         self.set_progress(now=self._progress["now"]+step, label="Initializing Superset...")
-        result = self.run_command(f"{superset_path} init", env)
+        result = self.run_command(f'"{superset_path}" init', env)
         if "error" in result:
             return result
 
         # Load examples (optional)
         print("Loading example data...")
-        self.set_progress(now=self._progress["now"]+step, label="Loading default example data...")
-        result = self.run_command(f"{superset_path} load_examples", env)
+        self.set_progress(now=self._progress["now"]+step, label="Loading default example data (this may take several minutes)...")
+        result = self.run_command(f'"{superset_path}" load_examples', env)
         if "error" in result:
-            return result
+            print(f"Warning: Failed to load examples: {result.get('error')}")
+            # We continue even if examples fail to load, as it is optional
 
         # Check if port is available
         print(f"Checking if port {port} is available...")
@@ -205,7 +217,7 @@ def setup_superset(self, port, python_path):
         print(f"Launching Superset on port {port}...")
         self.set_progress(now=self._progress["now"]+step, label="Launching Superset...")
         try:
-            subprocess.Popen(f"{superset_path} run -p {port}", shell=True, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            subprocess.Popen(f'"{superset_path}" run -p {port}', shell=True, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
         except subprocess.CalledProcessError as e:
             print(f"Error while running command: {f'{superset_path} run -p {port}'}")
             print(e.stderr)

pythonCode/modules/superset/launch.py

@@ -84,12 +84,15 @@ const SupersetDashboard = () => {
   async function launchSuperset() {
     let freePort = 8080 // in the future maybe we'll use getPort() from get-port package
     let pythonPath = await ipcRenderer.invoke("getBundledPythonEnvironment")
+    let appDataPath = await ipcRenderer.invoke("appGetPath", "userData")
 
     // Send the request to the backend
     let jsonToSend = {
       "port": freePort,
       "pythonPath": pythonPath,
+      "appDataPath": appDataPath,
     }
+    console.log("Launching superset with python path:", pythonPath)
     setLoading(true)
     requestBackend(
       port,
@@ -136,10 +139,12 @@ const SupersetDashboard = () => {
   async function createUser() {
     // get Python path
     let pythonPath = await ipcRenderer.invoke("getBundledPythonEnvironment")
+    let appDataPath = await ipcRenderer.invoke("appGetPath", "userData")
 
     // Send the request to the backend
     let jsonToSend = {
       "pythonPath": pythonPath,
+      "appDataPath": appDataPath,
       "username": newUserUsername,
       "password": newUserPassword,
       "firstname": newFirstName,
@@ -309,9 +314,20 @@ const SupersetDashboard = () => {
     return
   }
 
-  const getEnvPath = (pythonPath) => {
+  const getEnvPath = (pythonPath, appDataPath) => {
     if (!pythonPath) return ''
 
+    // If appDataPath is provided, use it to construct the path
+    if (appDataPath) {
+      const isWindows = os.platform() === "win32"
+      if (isWindows) {
+        return `${appDataPath}\\superset_env\\Scripts\\python.exe`
+      } else {
+        return `${appDataPath}/superset_env/bin/python`
+      }
+    }
+
+    // Fallback to old logic if appDataPath is not available (should not happen with new logic)
     const isWindows = os.platform() === "win32"
     if (isWindows) {
       const pathParts = pythonPath.split(/[\\/]/)
@@ -323,7 +339,8 @@ const SupersetDashboard = () => {
   }
 
   const getSupersetConfigPath = async (pythonPath) => {
-    const envPath = getEnvPath(pythonPath)
+    let appDataPath = await ipcRenderer.invoke("appGetPath", "userData")
+    const envPath = getEnvPath(pythonPath, appDataPath)
     if (!envPath) return ''
 
     const isWindows = os.platform() === "win32"