Merge pull request #434 from MEDomics-UdeS/scalability

MahdiAll99 · web-flow · commit 7ed271cb46ca · 2025-03-11T16:57:42.000-04:00
V1.4.0 merge
diff --git a/go_server/blueprints/input/input.go b/go_server/blueprints/input/input.go
@@ -1,6 +1,7 @@
 package input
 
 import (
+	"fmt"
 	Utils "go_module/src"
 	"log"
 )
@@ -26,6 +27,8 @@ func AddHandleFunc() {
 	Utils.CreateHandleFunc(prePath+"/get_subset_data/", handleGetSubsetData)
 	Utils.CreateHandleFunc(prePath+"/create_new_collection/", handleCreateNewCollection)
 	Utils.CreateHandleFunc(prePath+"/overwrite_collection/", handleOverwriteCollection)
+	Utils.CreateHandleFunc(prePath+"/overwrite_encoded_data", handleOverwriteEncodedData)
+	Utils.CreateHandleFunc(prePath+"/append_encoded_data", handleAppendEncodedData)
 	Utils.CreateHandleFunc(prePath+"/create_group_DB/", handleCreateGroupDB)
 }
 
@@ -234,6 +237,40 @@ func handleOverwriteCollection(jsonConfig string, id string) (string, error) {
 	return response, nil
 }
 
+func handleOverwriteEncodedData(jsonConfig string, id string) (string, error) {
+
+	response, err := Utils.StartPythonScripts(jsonConfig, "../pythonCode/modules/input/overwrite_encoded_data.py", id)
+	if err != nil {
+		log.Println("Error executing Python script:", err)
+		return "", err
+	}
+
+	// Log to verify response
+	log.Println("Python script response:", response)
+
+	if response == "" {
+		return "", fmt.Errorf("empty response from Python script")
+	}
+
+	return response, nil
+}
+
+func handleAppendEncodedData(jsonConfig string, id string) (string, error) {
+	response, err := Utils.StartPythonScripts(jsonConfig, "../pythonCode/modules/input/append_encoded_data.py", id)
+	if err != nil {
+		log.Println("Error executing Python script:", err)
+		return "", err
+	}
+
+	log.Println("Python script response:", response)
+
+	if response == "" {
+		return "", fmt.Errorf("empty response from Python script")
+	}
+
+	return response, nil
+}
+
 // handleCreateGroupDB handles the request to create the group for the DB
 // It returns the response from the python script
 func handleCreateGroupDB(jsonConfig string, id string) (string, error) {
diff --git a/go_server/main b/go_server/main
diff --git a/pythonCode/modules/input/append_encoded_data.py b/pythonCode/modules/input/append_encoded_data.py
@@ -0,0 +1,88 @@
+import json
+import os
+import sys
+from pathlib import Path
+
+sys.path.append(
+    str(Path(os.path.dirname(os.path.abspath(__file__))).parent.parent))
+from med_libs.GoExecutionScript import GoExecutionScript, parse_arguments
+from med_libs.server_utils import go_print
+from med_libs.mongodb_utils import connect_to_mongo
+
+# Parse arguments
+json_params_dict, id_ = parse_arguments()
+go_print("running script.py:" + id_)
+
+
+class GoExecScriptAppend(GoExecutionScript):
+    """
+    This class overwrites an existing MongoDB collection while keeping existing columns.
+
+    Args:
+        json_params: Input JSON parameters
+        _id: Request ID (optional)
+    """
+
+    def __init__(self, json_params: dict, _id: str = None):
+        super().__init__(json_params, _id)
+        self.results = {"status": "error", "message": "Process not completed."}  # Default response
+
+    def _custom_process(self, json_config: dict) -> dict:
+        """
+        Overwrites the specified collection with new data while preserving existing columns.
+        """
+        try:
+
+            if "collectionName" not in json_config or "data" not in json_config:
+                raise ValueError("Invalid JSON format: 'collectionName' or 'data' missing.")
+
+            # Set local variables
+            collection_name = json_config["collectionName"]
+            new_data = json_config["data"]
+
+
+            # Connect to MongoDB
+            db = connect_to_mongo()
+            collection = db[collection_name]
+
+     
+            existing_doc = collection.find_one({}, {"_id": 0})
+
+
+            if existing_doc:
+                all_keys = set(existing_doc.keys())
+            else:
+                all_keys = set()
+
+            
+            # Append new Data
+            for doc in new_data:
+                all_keys.update(doc.keys())
+
+      
+            temp_doc = {key: None for key in all_keys}  
+            temp_id = collection.insert_one(temp_doc).inserted_id
+
+ 
+            collection.delete_many({"_id": {"$ne": temp_id}})  
+            collection.insert_many(new_data)
+            collection.delete_one({"_id": temp_id})
+
+        # Return success
+            self.results = {
+                "status": "success",
+                "message": "Data overwritten successfully while keeping existing schema."
+            }
+
+        # Handle exceptions
+        except Exception as e:
+            self.results = {"status": "error", "message": str(e)}
+            go_print(f"Error: {str(e)}")
+
+        return self.results  
+
+
+
+if __name__ == "__main__":
+    script = GoExecScriptAppend(json_params_dict, id_)
+    script.start()  # Start the process and execute `_custom_process`
diff --git a/pythonCode/modules/input/overwrite_encoded_data.py b/pythonCode/modules/input/overwrite_encoded_data.py
@@ -0,0 +1,64 @@
+import json
+import os
+import sys
+from pathlib import Path
+
+sys.path.append(
+    str(Path(os.path.dirname(os.path.abspath(__file__))).parent.parent))
+from med_libs.GoExecutionScript import GoExecutionScript, parse_arguments
+from med_libs.mongodb_utils import connect_to_mongo
+from med_libs.server_utils import go_print
+
+json_params_dict, id_ = parse_arguments()
+go_print("running script.py:" + id_)
+
+
+
+
+class GoExecScriptOverwrite(GoExecutionScript):
+    """
+    This class overwrites data in a MongoDB collection.
+
+    Args:
+        json_params: Input JSON parameters
+        _id: Request ID (optional)
+    """
+
+    def __init__(self, json_params: dict, _id: str = None):
+        super().__init__(json_params, _id)
+        self.results = None  # Initially set to None
+
+    def _custom_process(self, json_config: dict) -> dict:
+        """
+        Overwrites the specified collection with new data.
+
+        Args:
+            json_config: Input JSON parameters
+        """
+        try:
+            # Set local variables
+            collection_name = json_config["collectionName"]
+            new_data = json_config["data"]
+
+            # Connect to MongoDB
+            db = connect_to_mongo()
+            collection = db[collection_name]
+
+            # Overwrite the data
+            go_print(f"Overwriting data in collection: {collection_name}")
+            collection.drop()
+            collection.insert_many(new_data)
+
+            # Return success
+            self.results = {"status": "success", "message": "Data overwritten successfully."}
+        except Exception as e:
+            # Handle exceptions
+            self.results = {"status": "error", "message": str(e)}
+            go_print(f"Error: {str(e)}")
+
+        return self.results
+
+
+# Start the script
+script = GoExecScriptOverwrite(json_params_dict, id_)
+script.start()
diff --git a/renderer/components/dbComponents/InputToolsComponent.jsx b/renderer/components/dbComponents/InputToolsComponent.jsx
@@ -8,6 +8,7 @@ import { DataContext } from "../workspace/dataContext"
 import BasicToolsDB from "./inputToolsDB/basicToolsDB"
 import DropDuplicatesToolsDB from "./inputToolsDB/dropDuplicatesToolsDB"
 import FeatureReductionToolsDB from "./inputToolsDB/featureReductionToolsDB/featureReductionToolsDB"
+import ConvertCategoricalColumnIntoNumericDB from "./inputToolsDB/convertCategoricalColumnIntoNumericDB"
 import GroupingTaggingToolsDB from "./inputToolsDB/groupingTaggingToolsDB"
 import HoldoutSetCreationToolsDB from "./inputToolsDB/holdoutSetCreationToolsDB"
 import MergeToolsDB from "./inputToolsDB/mergeToolsDB"
@@ -95,6 +96,9 @@ const InputToolsComponent = ({ exportOptions }) => {
           <Panel header="Basic Tools" toggleable collapsed={true}>
             <BasicToolsDB collectionSize={collectionSize} currentCollection={!collectionId ? null : collectionId} />
           </Panel>
+          <Panel header="Convert Categorical Column Into Numeric" toggleable collapsed={true}>
+            <ConvertCategoricalColumnIntoNumericDB exportOptions={exportOptions} currentCollection={!collectionId ? null : collectionId} />
+          </Panel>
           <Panel header="Drop Duplicates Tools" toggleable collapsed={true}>
             <DropDuplicatesToolsDB exportOptions={exportOptions} currentCollection={!collectionId ? null : collectionId} />
           </Panel>
diff --git a/renderer/components/dbComponents/dataTableFromDB.jsx b/renderer/components/dbComponents/dataTableFromDB.jsx
@@ -770,7 +770,7 @@ const DataTableFromDB = ({ data, tablePropsData, tablePropsColumn, isReadOnly })
     // Loop through each tag to find the group names for the current rowData
     rowTags.forEach(tag => {
       tag.data.forEach(item => {
-        if (item._id.toString() === rowData._id.toString()) { 
+        if (item?._id.toString() === rowData?._id.toString()) { 
           item.groupNames.forEach(groupName => groupNames.add(groupName)); 
         }
       });
@@ -844,7 +844,6 @@ const DataTableFromDB = ({ data, tablePropsData, tablePropsColumn, isReadOnly })
 
   return (
       <>
-      {console.log("TESDTY", rowTags[0])}
         <Dialog visible={viewMode} header="Preview changes" style={{ width: "80%", height: "80%" }} modal={true} onHide={() => onCancelChanges()}>
         <DataTable
           className="p-datatable-striped p-datatable-gridlines"
diff --git a/renderer/components/dbComponents/inputToolsDB/convertCategoricalColumnIntoNumericDB.jsx b/renderer/components/dbComponents/inputToolsDB/convertCategoricalColumnIntoNumericDB.jsx
