Skip to content

Commit bf787d4

Browse files
MahdiAll99MahdiAll99
authored
Merge pull request #430 from MEDomics-UdeS/scalability130
V1.3.0 Merge - Big files fix
2 parents ac2c411 + 00106ff commit bf787d4

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+1871
-1248
lines changed

‎.github/workflows/automaticBuilding.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ jobs:
7979
VERSION: ${{ steps.get-tag.outputs.VERSION }}
8080

8181
build-linux-ubuntu:
82-
runs-on: ubuntu-20.04
82+
runs-on: ubuntu-22.04
8383

8484
steps:
8585
- name: Checkout code

‎.github/workflows/automaticBuildingLinux.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ permissions:
77
contents: write
88
jobs:
99
build-linux-ubuntu:
10-
runs-on: ubuntu-20.04
10+
runs-on: ubuntu-22.04
1111

1212
steps:
1313
- name: Checkout code

‎go_server/blueprints/input/input.go‎

Lines changed: 70 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,14 @@ func AddHandleFunc() {
1919
Utils.CreateHandleFunc(prePath+"/compute_correlationsDB/", handleComputeCorrelationsDB)
2020
Utils.CreateHandleFunc(prePath+"/compute_spearmanDB/", handleComputeSpearmanDB)
2121
Utils.CreateHandleFunc(prePath+"/create_tags/", handleCreateTags)
22-
Utils.CreateHandleFunc(prePath+"/delete_tag_from_column/", handleDeleteTagFromColumn)
2322
Utils.CreateHandleFunc(prePath+"/handle_pkl/", handlePKL)
23+
Utils.CreateHandleFunc(prePath+"/delete_columns/", deleteColumns)
24+
Utils.CreateHandleFunc(prePath+"/transform_columns/", transformColumns)
25+
Utils.CreateHandleFunc(prePath+"/get_row_column_missing_values/", handleGetMissingValues)
26+
Utils.CreateHandleFunc(prePath+"/get_subset_data/", handleGetSubsetData)
27+
Utils.CreateHandleFunc(prePath+"/create_new_collection/", handleCreateNewCollection)
28+
Utils.CreateHandleFunc(prePath+"/overwrite_collection/", handleOverwriteCollection)
2429
Utils.CreateHandleFunc(prePath+"/create_group_DB/", handleCreateGroupDB)
25-
Utils.CreateHandleFunc(prePath+"/delete_row_tag_DB/", handleDeleteRowTagDB)
2630
}
2731

2832
// handleMerge handles the request to merge the datasets for the DB
@@ -146,47 +150,95 @@ func handleCreateTags(jsonConfig string, id string) (string, error) {
146150
return response, nil
147151
}
148152

149-
// handleDeleteTagFromColumn handles the request to tag deletion for the DB
153+
// handlePKL handles the request to handle the pkl file
150154
// It returns the response from the python script
151-
func handleDeleteTagFromColumn(jsonConfig string, id string) (string, error) {
152-
log.Println("Compute Tag Deletion", id)
153-
response, err := Utils.StartPythonScripts(jsonConfig, "../pythonCode/modules/input/delete_tag_from_column.py", id)
155+
func handlePKL(jsonConfig string, id string) (string, error) {
156+
log.Println("handling .pkl filetype", id)
157+
response, err := Utils.StartPythonScripts(jsonConfig, "../pythonCode/modules/input/handle_pkl.py", id)
154158
Utils.RemoveIdFromScripts(id)
155159
if err != nil {
156160
return "", err
157161
}
158162
return response, nil
159163
}
160164

161-
// handlePKL handles the request to handle the pkl file
165+
// deleteColumns handles the request to delete columns from the DB
162166
// It returns the response from the python script
163-
func handlePKL(jsonConfig string, id string) (string, error) {
164-
log.Println("handling .pkl filetype", id)
165-
response, err := Utils.StartPythonScripts(jsonConfig, "../pythonCode/modules/input/handle_pkl.py", id)
167+
func deleteColumns(jsonConfig string, id string) (string, error) {
168+
log.Println("Deleting Columns", id)
169+
response, err := Utils.StartPythonScripts(jsonConfig, "../pythonCode/modules/input/delete_columns.py", id)
166170
Utils.RemoveIdFromScripts(id)
167171
if err != nil {
168172
return "", err
169173
}
170174
return response, nil
171175
}
172176

173-
// handleCreateGroupDB handles the request to create the group for the DB
177+
// transformColumns handles the request to transform columns from the DB
174178
// It returns the response from the python script
175-
func handleCreateGroupDB(jsonConfig string, id string) (string, error) {
176-
log.Println("Create Group DB", id)
177-
response, err := Utils.StartPythonScripts(jsonConfig, "../pythonCode/modules/input/create_group_DB.py", id)
179+
func transformColumns(jsonConfig string, id string) (string, error) {
180+
log.Println("Transforming Columns", id)
181+
response, err := Utils.StartPythonScripts(jsonConfig, "../pythonCode/modules/input/transform_columns.py", id)
182+
Utils.RemoveIdFromScripts(id)
183+
if err != nil {
184+
return "", err
185+
}
186+
return response, nil
187+
}
188+
189+
// handleGetMissingValues handles the request to get the missing values from the DB
190+
// It returns the response from the python script
191+
func handleGetMissingValues(jsonConfig string, id string) (string, error) {
192+
log.Println("Getting Missing Values", id)
193+
response, err := Utils.StartPythonScripts(jsonConfig, "../pythonCode/modules/input/get_row_column_missing_values.py", id)
194+
Utils.RemoveIdFromScripts(id)
195+
if err != nil {
196+
return "", err
197+
}
198+
return response, nil
199+
}
200+
201+
// handleGetSubsetData handles the request to get the subset data from the DB
202+
// It returns the response from the python script
203+
func handleGetSubsetData(jsonConfig string, id string) (string, error) {
204+
log.Println("Getting Subset Data", id)
205+
response, err := Utils.StartPythonScripts(jsonConfig, "../pythonCode/modules/input/get_subset_data.py", id)
206+
Utils.RemoveIdFromScripts(id)
207+
if err != nil {
208+
return "", err
209+
}
210+
return response, nil
211+
}
212+
213+
// handleCreateNewCollection handles the request to create a new collection from the DB
214+
// It returns the response from the python script
215+
func handleCreateNewCollection(jsonConfig string, id string) (string, error) {
216+
log.Println("Creating New Collection", id)
217+
response, err := Utils.StartPythonScripts(jsonConfig, "../pythonCode/modules/input/create_new_collection.py", id)
178218
Utils.RemoveIdFromScripts(id)
179219
if err != nil {
180220
return "", err
181221
}
182222
return response, nil
183223
}
184224

185-
// deleteRowTagDB handles the request to delete the row tag for the DB
225+
// handleOverwriteCollection handles the request to overwrite the collection from the DB
186226
// It returns the response from the python script
187-
func handleDeleteRowTagDB(jsonConfig string, id string) (string, error) {
188-
log.Println("Delete Row Tag DB", id)
189-
response, err := Utils.StartPythonScripts(jsonConfig, "../pythonCode/modules/input/delete_row_tag_DB.py", id)
227+
func handleOverwriteCollection(jsonConfig string, id string) (string, error) {
228+
log.Println("Overwriting Collection", id)
229+
response, err := Utils.StartPythonScripts(jsonConfig, "../pythonCode/modules/input/overwrite_collection.py", id)
230+
Utils.RemoveIdFromScripts(id)
231+
if err != nil {
232+
return "", err
233+
}
234+
return response, nil
235+
}
236+
237+
// handleCreateGroupDB handles the request to create the group for the DB
238+
// It returns the response from the python script
239+
func handleCreateGroupDB(jsonConfig string, id string) (string, error) {
240+
log.Println("Create Group DB", id)
241+
response, err := Utils.StartPythonScripts(jsonConfig, "../pythonCode/modules/input/create_group_DB.py", id)
190242
Utils.RemoveIdFromScripts(id)
191243
if err != nil {
192244
return "", err

‎go_server/main‎

-6.71 MB
Binary file not shown.

‎go_server/main.exe‎

804 KB
Binary file not shown.

‎main/background.js‎

Lines changed: 47 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -172,27 +172,27 @@ if (isProd) {
172172
console.log("process.resourcesPath: ", process.resourcesPath)
173173
console.log(MEDconfig.runServerAutomatically ? "Server will start automatically here (in background of the application)" : "Server must be started manually")
174174
let bundledPythonPath = getBundledPythonEnvironment()
175-
if (MEDconfig.runServerAutomatically && bundledPythonPath !== null) {
175+
if (MEDconfig.runServerAutomatically && bundledPythonPath !== null) {
176176
// Find the bundled python environment
177177
if (bundledPythonPath !== null) {
178178
runServer(isProd, serverPort, serverProcess, serverState, bundledPythonPath)
179-
.then((process) => {
180-
serverProcess = process
181-
console.log("Server process started: ", serverProcess)
182-
})
183-
.catch((err) => {
184-
console.error("Failed to start server: ", err)
185-
})
179+
.then((process) => {
180+
serverProcess = process
181+
console.log("Server process started: ", serverProcess)
182+
})
183+
.catch((err) => {
184+
console.error("Failed to start server: ", err)
185+
})
186186
}
187187
} else {
188188
//**** NO SERVER ****//
189189
findAvailablePort(MEDconfig.defaultPort)
190-
.then((port) => {
191-
serverPort = port
192-
})
193-
.catch((err) => {
194-
console.error(err)
195-
})
190+
.then((port) => {
191+
serverPort = port
192+
})
193+
.catch((err) => {
194+
console.error(err)
195+
})
196196
}
197197
const menu = Menu.buildFromTemplate(menuTemplate)
198198
Menu.setApplicationMenu(menu)
@@ -226,18 +226,12 @@ if (isProd) {
226226
// Kill the process on the port
227227
// killProcessOnPort(serverPort)
228228
} else if (process.platform === "darwin") {
229-
await new Promise((resolve, reject) => {
230-
exec("pkill -f mongod", (error, stdout, stderr) => {
231-
if (error) {
232-
console.error(`exec error: ${error}`)
233-
reject(error)
229+
await new Promise((resolve) => {
230+
exec("pkill -f mongod", (error, stdout, stderr) => {
231+
resolve()
232+
})
234233
}
235-
console.log(`stdout: ${stdout}`)
236-
console.error(`stderr: ${stderr}`)
237-
resolve()
238-
})
239-
}
240-
)
234+
)
241235
} else {
242236
try {
243237
execSync("killall mongod")
@@ -363,16 +357,16 @@ if (isProd) {
363357
}
364358
if (MEDconfig.runServerAutomatically) {
365359
runServer(isProd, serverPort, serverProcess, serverState, condaPath)
366-
.then((process) => {
367-
serverProcess = process
368-
console.log(`success: ${serverState.serverIsRunning}`)
369-
return serverState.serverIsRunning
370-
})
371-
.catch((err) => {
372-
console.error("Failed to start server: ", err)
373-
serverState.serverIsRunning = false
374-
return false
375-
})
360+
.then((process) => {
361+
serverProcess = process
362+
console.log(`success: ${serverState.serverIsRunning}`)
363+
return serverState.serverIsRunning
364+
})
365+
.catch((err) => {
366+
console.error("Failed to start server: ", err)
367+
serverState.serverIsRunning = false
368+
return false
369+
})
376370
}
377371
return serverState.serverIsRunning
378372
})
@@ -576,7 +570,7 @@ function startMongoDB(workspacePath) {
576570
if (fs.existsSync(getMongoDBPath())) {
577571
mongoProcess = spawn(getMongoDBPath(), ["--config", mongoConfigPath])
578572
} else {
579-
mongoProcess = spawn("/opt/homebrew/Cellar/mongodb-community/7.0.12/bin/mongod", ["--config", mongoConfigPath], { shell: true })
573+
mongoProcess = spawn("/opt/homebrew/Cellar/mongodb-community/7.0.12/bin/mongod", ["--config", mongoConfigPath], { shell: true })
580574
}
581575
}
582576
mongoProcess.stdout.on("data", (data) => {
@@ -654,28 +648,28 @@ export function getMongoDBPath() {
654648
return null
655649
} else if (process.platform === "darwin") {
656650
// Check if it is installed in the .medomics directory
657-
const binPath = path.join(process.env.HOME, ".medomics", "mongodb", "bin", "mongod")
658-
if (fs.existsSync(binPath)) {
659-
console.log("mongod found in .medomics directory")
660-
return binPath
661-
}
651+
const binPath = path.join(process.env.HOME, ".medomics", "mongodb", "bin", "mongod")
652+
if (fs.existsSync(binPath)) {
653+
console.log("mongod found in .medomics directory")
654+
return binPath
655+
}
662656
if (process.env.NODE_ENV !== "production") {
663657

664-
// Check if mongod is in the process.env.PATH
665-
const paths = process.env.PATH.split(path.delimiter)
666-
for (let i = 0; i < paths.length; i++) {
667-
const binPath = path.join(paths[i], "mongod")
658+
// Check if mongod is in the process.env.PATH
659+
const paths = process.env.PATH.split(path.delimiter)
660+
for (let i = 0; i < paths.length; i++) {
661+
const binPath = path.join(paths[i], "mongod")
662+
if (fs.existsSync(binPath)) {
663+
console.log("mongod found in PATH")
664+
return binPath
665+
}
666+
}
667+
// Check if mongod is in the default installation path on macOS - /usr/local/bin/mongod
668+
const binPath = "/usr/local/bin/mongod"
668669
if (fs.existsSync(binPath)) {
669-
console.log("mongod found in PATH")
670670
return binPath
671671
}
672672
}
673-
// Check if mongod is in the default installation path on macOS - /usr/local/bin/mongod
674-
const binPath = "/usr/local/bin/mongod"
675-
if (fs.existsSync(binPath)) {
676-
return binPath
677-
}
678-
}
679673
console.error("mongod not found")
680674
return null
681675
} else if (process.platform === "linux") {
@@ -693,7 +687,7 @@ export function getMongoDBPath() {
693687
return "/usr/bin/mongod"
694688
}
695689
console.error("mongod not found in /usr/bin/mongod")
696-
690+
697691
if (fs.existsSync("/home/"+process.env.USER+"/.medomics/mongodb/bin/mongod")) {
698692
return "/home/"+process.env.USER+"/.medomics/mongodb/bin/mongod"
699693
}

‎package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@
9696
"electron-store": "^8.1.0",
9797
"eslint-config-next": "^13.5.3",
9898
"eslint-config-prettier": "^8.10.0",
99-
"eslint-plugin-prettier": "^5.1.3",
99+
"eslint-plugin-prettier": "^5.2.3",
100100
"file-saver": "^2.0.5",
101101
"flexlayout-react": "^0.7.7",
102102
"html-react-parser": "^3.0.12",
@@ -155,7 +155,7 @@
155155
"eslint-plugin-react": "^7.32.2",
156156
"next": "^13.5.3",
157157
"nextron": "^8.5.0",
158-
"prettier": "^3.3.1",
158+
"prettier": "^3.4.2",
159159
"react": "^18.2.0",
160160
"react-bootstrap": "^2.7.1",
161161
"react-dom": "^18.2.0"

‎pythonCode/modules/input/apply_pcaDB.py‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,17 @@
11
import json
2-
import sys
3-
import numpy as np
42
import os
5-
import pandas as pd
3+
import sys
64
from pathlib import Path
5+
6+
import pandas as pd
7+
78
sys.path.append(
89
str(Path(os.path.dirname(os.path.abspath(__file__))).parent.parent))
10+
911
from med_libs.GoExecutionScript import GoExecutionScript, parse_arguments
12+
from med_libs.mongodb_utils import connect_to_mongo
1013
from med_libs.server_utils import go_print
1114

12-
# To deal with the DB
13-
from pymongo import MongoClient
14-
import math
15-
1615
json_params_dict, id_ = parse_arguments()
1716
go_print("running script.py:" + id_)
1817

@@ -45,13 +44,11 @@ def _custom_process(self, json_config: dict) -> dict:
4544
keep_unselected_columns = json_config["keepUnselectedColumns"]
4645
overwrite = json_config["overwrite"]
4746
collection_name = json_config["collectionName"]
48-
database_name = json_config["databaseName"]
4947
new_collection_name = json_config["newCollectionName"]
5048
transformationCollection = json_config["transformationCollection"]
5149

5250
# Connect to MongoDB
53-
client = MongoClient('localhost', 54017)
54-
db = client[database_name]
51+
db = connect_to_mongo()
5552
collection = db[collection_name]
5653

5754
# Fetch data and convert to DataFrame

0 commit comments

Comments
 (0)