From fa4dc3466b48cfca16060a67b56a37218aea9055 Mon Sep 17 00:00:00 2001 From: DImuthuUpe Date: Fri, 28 Nov 2025 17:52:53 -0500 Subject: [PATCH 1/6] Adding support to clean up experiment working directory on cluster --- .../init/07-cleanup-strategy-migration.sql | 4 +++ .../airavata/agents/api/AgentAdaptor.java | 2 ++ .../agents/api/StorageResourceAdaptor.java | 2 ++ .../helix/adaptor/SSHJAgentAdaptor.java | 24 +++++++++++++ .../helix/agent/ssh/SshAgentAdaptor.java | 34 +++++++++++++++++++ .../helix/impl/task/AiravataTask.java | 7 ++++ .../airavata/helix/impl/task/TaskContext.java | 13 +++++++ .../impl/task/completing/CompletingTask.java | 19 +++++++++++ .../entities/expcatalog/ExperimentEntity.java | 14 ++++++++ .../data-models/experiment_model.thrift | 10 +++++- 10 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 .devcontainer/database_scripts/init/07-cleanup-strategy-migration.sql diff --git a/.devcontainer/database_scripts/init/07-cleanup-strategy-migration.sql b/.devcontainer/database_scripts/init/07-cleanup-strategy-migration.sql new file mode 100644 index 0000000000..7214580815 --- /dev/null +++ b/.devcontainer/database_scripts/init/07-cleanup-strategy-migration.sql @@ -0,0 +1,4 @@ +USE experiment_catalog; + +-- Add cleanAfterStaged flag to APPLICATION_INTERFACE +ALTER TABLE EXPERIMENT ADD COLUMN IF NOT EXISTS CLEANUP_STRATEGY VARCHAR(255) DEFAULT 'NONE'; diff --git a/airavata-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java b/airavata-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java index 36baea7060..575e2c396f 100644 --- a/airavata-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java +++ b/airavata-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java @@ -43,6 +43,8 @@ public interface AgentAdaptor { void createDirectory(String path, boolean recursive) throws AgentException; + void deleteDirectory(String path) throws AgentException; + void uploadFile(String localFile, String remoteFile) throws AgentException; void uploadFile(InputStream localInStream, FileMetadata metadata, String remoteFile) throws AgentException; diff --git a/airavata-api/src/main/java/org/apache/airavata/agents/api/StorageResourceAdaptor.java b/airavata-api/src/main/java/org/apache/airavata/agents/api/StorageResourceAdaptor.java index 845dc04f78..83ee95945c 100644 --- a/airavata-api/src/main/java/org/apache/airavata/agents/api/StorageResourceAdaptor.java +++ b/airavata-api/src/main/java/org/apache/airavata/agents/api/StorageResourceAdaptor.java @@ -41,6 +41,8 @@ public void downloadFile(String remoteFile, OutputStream localOutStream, FileMet public void createDirectory(String path, boolean recursive) throws AgentException; + public void deleteDirectory(String path) throws AgentException; + public List listDirectory(String path) throws AgentException; public Boolean doesFileExist(String filePath) throws AgentException; diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java b/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java index 2845afb1ef..b687c3093d 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java @@ -277,6 +277,30 @@ public void createDirectory(String path, boolean recursive) throws AgentExceptio } } + @Override + public void deleteDirectory(String path) throws AgentException { + SFTPClientWrapper sftpClient = null; + try { + sftpClient = sshjClient.newSFTPClientWrapper(); + sftpClient.rmdir(path); + } catch (Exception e) { + if (e instanceof ConnectionException) { + Optional.ofNullable(sftpClient).ifPresent(ft -> ft.setErrored(true)); + } + logger.error("Error while deleting directory " + path, e); + throw new AgentException(e); + + } finally { + Optional.ofNullable(sftpClient).ifPresent(client -> { + try { + client.close(); + } catch (IOException e) { + // Ignore + } + }); + } + } + @Override public void uploadFile(String localFile, String remoteFile) throws AgentException { SCPFileTransferWrapper fileTransfer = null; diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java b/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java index 8ee4237fd7..20c9d27387 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java @@ -182,6 +182,40 @@ public void createDirectory(String path, boolean recursive) throws AgentExceptio } } + @Override + public void deleteDirectory(String path) throws AgentException { + String command = "rm -rf " + path; + ChannelExec channelExec = null; + try { + channelExec = (ChannelExec) session.openChannel("exec"); + StandardOutReader stdOutReader = new StandardOutReader(); + + channelExec.setCommand(command); + InputStream out = channelExec.getInputStream(); + InputStream err = channelExec.getErrStream(); + channelExec.connect(); + + stdOutReader.readStdOutFromStream(out); + stdOutReader.readStdErrFromStream(err); + + if (stdOutReader.getStdError() != null && stdOutReader.getStdError().contains("mkdir:")) { + throw new AgentException(stdOutReader.getStdError()); + } + } catch (JSchException e) { + System.out.println("Unable to retrieve command output. Command - " + command + " on server - " + + session.getHost() + ":" + session.getPort() + " connecting user name - " + + session.getUserName()); + throw new AgentException(e); + } catch (IOException e) { + logger.error("Failed to delete directory " + path, e); + throw new AgentException("Failed to delete directory " + path, e); + } finally { + if (channelExec != null) { + channelExec.disconnect(); + } + } + } + public void uploadFile(String localFile, String remoteFile) throws AgentException { FileInputStream fis; diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java index f5f06c35ac..142ffb13b1 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java @@ -70,6 +70,7 @@ public abstract class AiravataTask extends AbstractTask { private static Publisher statusPublisher; private ProcessModel processModel; + private ExperimentModel experimentModel; private ComputeResourceDescription computeResourceDescription; private TaskContext taskContext; private String taskName; @@ -519,6 +520,7 @@ protected void loadContext() throws TaskOnFailException { try { logger.info("Loading context for task " + getTaskId()); processModel = getRegistryServiceClient().getProcess(processId); + experimentModel = getRegistryServiceClient().getExperiment(experimentId); this.computeResourceDescription = getRegistryServiceClient().getComputeResource(this.processModel.getComputeResourceId()); @@ -527,6 +529,7 @@ protected void loadContext() throws TaskOnFailException { getProcessId(), getGatewayId(), getTaskId()) .setRegistryClient(getRegistryServiceClient()) .setProfileClient(getUserProfileClient()) + .setExperimentModel(getExperimentModel()) .setProcessModel(getProcessModel()); this.taskContext = taskContextBuilder.build(); @@ -603,6 +606,10 @@ protected ProcessModel getProcessModel() { return processModel; } + protected ExperimentModel getExperimentModel() { + return experimentModel; + } + public void setSkipAllStatusPublish(boolean skipAllStatusPublish) { this.skipAllStatusPublish = skipAllStatusPublish; } diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/TaskContext.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/TaskContext.java index 5bab41ae8f..46b38fb0aa 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/TaskContext.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/TaskContext.java @@ -56,6 +56,7 @@ import org.apache.airavata.model.application.io.OutputDataObjectType; import org.apache.airavata.model.data.movement.DataMovementInterface; import org.apache.airavata.model.data.movement.DataMovementProtocol; +import org.apache.airavata.model.experiment.ExperimentModel; import org.apache.airavata.model.job.JobModel; import org.apache.airavata.model.process.ProcessModel; import org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel; @@ -91,6 +92,7 @@ public class TaskContext { private String gatewayId; private String taskId; + private ExperimentModel experimentModel; private ProcessModel processModel; private JobModel jobModel; private Object subTaskModel = null; @@ -176,6 +178,10 @@ public void setProcessModel(ProcessModel processModel) { this.processModel = processModel; } + public void setExperimentModel(ExperimentModel experimentModel) { + this.experimentModel = experimentModel; + } + public String getWorkingDir() throws Exception { if (workingDir == null) { if (processModel.getProcessResourceSchedule().getStaticWorkingDir() != null) { @@ -1002,6 +1008,7 @@ public static class TaskContextBuilder { private RegistryService.Client registryClient; private UserProfileService.Client profileClient; private ProcessModel processModel; + private ExperimentModel experimentModel; @SuppressWarnings("WeakerAccess") public TaskContextBuilder(String processId, String gatewayId, String taskId) throws Exception { @@ -1018,6 +1025,11 @@ public TaskContextBuilder setProcessModel(ProcessModel processModel) { return this; } + public TaskContextBuilder setExperimentModel(ExperimentModel experimentModel) { + this.experimentModel = experimentModel; + return this; + } + public TaskContextBuilder setRegistryClient(RegistryService.Client registryClient) { this.registryClient = registryClient; return this; @@ -1040,6 +1052,7 @@ public TaskContext build() throws Exception { TaskContext ctx = new TaskContext(processId, gatewayId, taskId); ctx.setRegistryClient(registryClient); ctx.setProcessModel(processModel); + ctx.setExperimentModel(experimentModel); ctx.setProfileClient(profileClient); return ctx; } diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/completing/CompletingTask.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/completing/CompletingTask.java index ca0c49ea4c..41ef2ef02d 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/completing/CompletingTask.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/completing/CompletingTask.java @@ -19,10 +19,12 @@ */ package org.apache.airavata.helix.impl.task.completing; +import org.apache.airavata.agents.api.AgentAdaptor; import org.apache.airavata.helix.impl.task.AiravataTask; import org.apache.airavata.helix.impl.task.TaskContext; import org.apache.airavata.helix.task.api.TaskHelper; import org.apache.airavata.helix.task.api.annotation.TaskDef; +import org.apache.airavata.model.experiment.ExperimentCleanupStrategy; import org.apache.airavata.model.status.ProcessState; import org.apache.helix.task.TaskResult; import org.slf4j.Logger; @@ -39,6 +41,23 @@ public TaskResult onRun(TaskHelper helper, TaskContext taskContext) { logger.info("Process " + getProcessId() + " successfully completed"); saveAndPublishProcessStatus(ProcessState.COMPLETED); cleanup(); + + try { + if (getExperimentModel().getCleanUpStrategy() == ExperimentCleanupStrategy.ALWAYS) { + AgentAdaptor adaptor = helper + .getAdaptorSupport() + .fetchAdaptor( + getTaskContext().getGatewayId(), + getTaskContext().getComputeResourceId(), + getTaskContext().getJobSubmissionProtocol(), + getTaskContext().getComputeResourceCredentialToken(), + getTaskContext().getComputeResourceLoginUserName()); + logger.info("Cleaning up the working directory {}", taskContext.getWorkingDir()); + //adaptor.deleteDirectory(getTaskContext().getWorkingDir()); + } + } catch (Exception e) { + logger.error("Failed clean up experiment " + getExperimentId(), e); + } return onSuccess("Process " + getProcessId() + " successfully completed"); } diff --git a/airavata-api/src/main/java/org/apache/airavata/registry/core/entities/expcatalog/ExperimentEntity.java b/airavata-api/src/main/java/org/apache/airavata/registry/core/entities/expcatalog/ExperimentEntity.java index 848c688e1b..3b0a024419 100644 --- a/airavata-api/src/main/java/org/apache/airavata/registry/core/entities/expcatalog/ExperimentEntity.java +++ b/airavata-api/src/main/java/org/apache/airavata/registry/core/entities/expcatalog/ExperimentEntity.java @@ -23,6 +23,8 @@ import java.io.Serializable; import java.sql.Timestamp; import java.util.List; + +import org.apache.airavata.model.experiment.ExperimentCleanupStrategy; import org.apache.airavata.model.experiment.ExperimentType; /** @@ -47,6 +49,10 @@ public class ExperimentEntity implements Serializable { @Enumerated(EnumType.STRING) public ExperimentType experimentType; + @Column(name = "CLEANUP_STRATEGY") + @Enumerated(EnumType.STRING) + public ExperimentCleanupStrategy cleanupStrategy; + @Column(name = "USER_NAME") public String userName; @@ -264,6 +270,14 @@ public void setExperimentStatus(List experimentStatus) { this.experimentStatus = experimentStatus; } + public ExperimentCleanupStrategy getCleanupStrategy() { + return cleanupStrategy; + } + + public void setCleanupStrategy(ExperimentCleanupStrategy cleanupStrategy) { + this.cleanupStrategy = cleanupStrategy; + } + public List getProcesses() { return processes; } diff --git a/thrift-interface-descriptions/data-models/experiment_model.thrift b/thrift-interface-descriptions/data-models/experiment_model.thrift index 07ec759fe1..6d3ec9ebe2 100644 --- a/thrift-interface-descriptions/data-models/experiment_model.thrift +++ b/thrift-interface-descriptions/data-models/experiment_model.thrift @@ -73,6 +73,13 @@ struct UserConfigurationDataModel { 13: optional list autoScheduledCompResourceSchedulingList, } +enum ExperimentCleanupStrategy { + NONE, + ALWAYS, + ONLY_COMPLETED, + ONLY_FAILED +} + /** * A structure holding the experiment metadata and its child models. * @@ -110,7 +117,8 @@ struct ExperimentModel { 17: optional list experimentStatus, 18: optional list errors, 19: optional list processes, - 20: optional workflow_model.AiravataWorkflow workflow + 20: optional workflow_model.AiravataWorkflow workflow, + 21: optional ExperimentCleanupStrategy cleanUpStrategy = ExperimentCleanupStrategy.NONE } struct ExperimentSummaryModel { From 6e24c6c6a768ce4e60ce34d78197e0d79ef2bc02 Mon Sep 17 00:00:00 2001 From: DImuthuUpe Date: Fri, 28 Nov 2025 17:55:32 -0500 Subject: [PATCH 2/6] Adding commented delete dir functionality --- .../airavata/helix/impl/task/completing/CompletingTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/completing/CompletingTask.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/completing/CompletingTask.java index 41ef2ef02d..2599ef94a5 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/completing/CompletingTask.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/completing/CompletingTask.java @@ -53,7 +53,7 @@ public TaskResult onRun(TaskHelper helper, TaskContext taskContext) { getTaskContext().getComputeResourceCredentialToken(), getTaskContext().getComputeResourceLoginUserName()); logger.info("Cleaning up the working directory {}", taskContext.getWorkingDir()); - //adaptor.deleteDirectory(getTaskContext().getWorkingDir()); + adaptor.deleteDirectory(getTaskContext().getWorkingDir()); } } catch (Exception e) { logger.error("Failed clean up experiment " + getExperimentId(), e); From c2fdce049fc7c114100319dd3be04de164adeb45 Mon Sep 17 00:00:00 2001 From: lahiruj Date: Sat, 29 Nov 2025 06:42:29 -0500 Subject: [PATCH 3/6] escaped the directory path, generated python thrift stubs, and updated the java styles --- .../init/07-cleanup-strategy-migration.sql | 2 +- .../helix/adaptor/SSHJAgentAdaptor.java | 5 +++- .../helix/agent/ssh/SshAgentAdaptor.java | 20 ++++++++++----- .../helix/impl/task/AiravataTask.java | 2 +- .../impl/task/completing/CompletingTask.java | 3 +-- .../entities/expcatalog/ExperimentEntity.java | 1 - .../airavata/model/experiment/ttypes.py | 25 ++++++++++++++++++- 7 files changed, 45 insertions(+), 13 deletions(-) diff --git a/.devcontainer/database_scripts/init/07-cleanup-strategy-migration.sql b/.devcontainer/database_scripts/init/07-cleanup-strategy-migration.sql index 7214580815..4b40dbd7ed 100644 --- a/.devcontainer/database_scripts/init/07-cleanup-strategy-migration.sql +++ b/.devcontainer/database_scripts/init/07-cleanup-strategy-migration.sql @@ -1,4 +1,4 @@ USE experiment_catalog; --- Add cleanAfterStaged flag to APPLICATION_INTERFACE +-- Add cleanupStrategy flag to EXPERIMENT ALTER TABLE EXPERIMENT ADD COLUMN IF NOT EXISTS CLEANUP_STRATEGY VARCHAR(255) DEFAULT 'NONE'; diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java b/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java index b687c3093d..495f9476d4 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java @@ -279,6 +279,9 @@ public void createDirectory(String path, boolean recursive) throws AgentExceptio @Override public void deleteDirectory(String path) throws AgentException { + if (path == null || path.trim().isEmpty()) { + throw new AgentException("Directory path cannot be null or empty"); + } SFTPClientWrapper sftpClient = null; try { sftpClient = sshjClient.newSFTPClientWrapper(); @@ -287,7 +290,7 @@ public void deleteDirectory(String path) throws AgentException { if (e instanceof ConnectionException) { Optional.ofNullable(sftpClient).ifPresent(ft -> ft.setErrored(true)); } - logger.error("Error while deleting directory " + path, e); + logger.error("Error while deleting directory {}", path, e); throw new AgentException(e); } finally { diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java b/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java index 20c9d27387..f2da8f8a7a 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java @@ -184,7 +184,11 @@ public void createDirectory(String path, boolean recursive) throws AgentExceptio @Override public void deleteDirectory(String path) throws AgentException { - String command = "rm -rf " + path; + if (path == null || path.trim().isEmpty()) { + throw new AgentException("Directory path cannot be null or empty"); + } + String escapedPath = path.replace("'", "'\"'\"'"); + String command = "rm -rf '" + escapedPath + "'"; ChannelExec channelExec = null; try { channelExec = (ChannelExec) session.openChannel("exec"); @@ -198,16 +202,20 @@ public void deleteDirectory(String path) throws AgentException { stdOutReader.readStdOutFromStream(out); stdOutReader.readStdErrFromStream(err); - if (stdOutReader.getStdError() != null && stdOutReader.getStdError().contains("mkdir:")) { + if (stdOutReader.getStdError() != null && stdOutReader.getStdError().contains("rm:")) { throw new AgentException(stdOutReader.getStdError()); } } catch (JSchException e) { - System.out.println("Unable to retrieve command output. Command - " + command + " on server - " - + session.getHost() + ":" + session.getPort() + " connecting user name - " - + session.getUserName()); + logger.error( + "Unable to retrieve command output. Command - {} on server - {}:{} connecting user name - {}", + command, + session.getHost(), + session.getPort(), + session.getUserName(), + e); throw new AgentException(e); } catch (IOException e) { - logger.error("Failed to delete directory " + path, e); + logger.error("Failed to delete directory {}", path, e); throw new AgentException("Failed to delete directory " + path, e); } finally { if (channelExec != null) { diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java index 142ffb13b1..97dec9e976 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java @@ -70,7 +70,7 @@ public abstract class AiravataTask extends AbstractTask { private static Publisher statusPublisher; private ProcessModel processModel; - private ExperimentModel experimentModel; + private ExperimentModel experimentModel; private ComputeResourceDescription computeResourceDescription; private TaskContext taskContext; private String taskName; diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/completing/CompletingTask.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/completing/CompletingTask.java index 2599ef94a5..83528fc366 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/completing/CompletingTask.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/completing/CompletingTask.java @@ -44,8 +44,7 @@ public TaskResult onRun(TaskHelper helper, TaskContext taskContext) { try { if (getExperimentModel().getCleanUpStrategy() == ExperimentCleanupStrategy.ALWAYS) { - AgentAdaptor adaptor = helper - .getAdaptorSupport() + AgentAdaptor adaptor = helper.getAdaptorSupport() .fetchAdaptor( getTaskContext().getGatewayId(), getTaskContext().getComputeResourceId(), diff --git a/airavata-api/src/main/java/org/apache/airavata/registry/core/entities/expcatalog/ExperimentEntity.java b/airavata-api/src/main/java/org/apache/airavata/registry/core/entities/expcatalog/ExperimentEntity.java index 3b0a024419..0a208ecf8c 100644 --- a/airavata-api/src/main/java/org/apache/airavata/registry/core/entities/expcatalog/ExperimentEntity.java +++ b/airavata-api/src/main/java/org/apache/airavata/registry/core/entities/expcatalog/ExperimentEntity.java @@ -23,7 +23,6 @@ import java.io.Serializable; import java.sql.Timestamp; import java.util.List; - import org.apache.airavata.model.experiment.ExperimentCleanupStrategy; import org.apache.airavata.model.experiment.ExperimentType; diff --git a/dev-tools/airavata-python-sdk/airavata/model/experiment/ttypes.py b/dev-tools/airavata-python-sdk/airavata/model/experiment/ttypes.py index abbcfe59b7..93d36e2732 100644 --- a/dev-tools/airavata-python-sdk/airavata/model/experiment/ttypes.py +++ b/dev-tools/airavata-python-sdk/airavata/model/experiment/ttypes.py @@ -51,6 +51,14 @@ class ProjectSearchFields(IntEnum): +class ExperimentCleanupStrategy(IntEnum): + NONE = 0 + ALWAYS = 1 + ONLY_COMPLETED = 2 + ONLY_FAILED = 3 + + + class UserConfigurationDataModel(object): """ A structure holding the experiment configuration. @@ -298,12 +306,13 @@ class ExperimentModel(object): - errors - processes - workflow + - cleanUpStrategy """ thrift_spec: typing.Any = None - def __init__(self, experimentId: str = "DO_NOT_SET_AT_CLIENTS", projectId: str = None, gatewayId: str = None, experimentType: ExperimentType = ExperimentType.SINGLE_APPLICATION, userName: str = None, experimentName: str = None, creationTime: typing.Optional[int] = None, description: typing.Optional[str] = None, executionId: typing.Optional[str] = None, gatewayExecutionId: typing.Optional[str] = None, gatewayInstanceId: typing.Optional[str] = None, enableEmailNotification: typing.Optional[bool] = None, emailAddresses: typing.Optional[list[str]] = None, userConfigurationData: typing.Optional[UserConfigurationDataModel] = None, experimentInputs: typing.Optional[list[airavata.model.application.io.ttypes.InputDataObjectType]] = None, experimentOutputs: typing.Optional[list[airavata.model.application.io.ttypes.OutputDataObjectType]] = None, experimentStatus: typing.Optional[list[airavata.model.status.ttypes.ExperimentStatus]] = None, errors: typing.Optional[list[airavata.model.commons.ttypes.ErrorModel]] = None, processes: typing.Optional[list[airavata.model.process.ttypes.ProcessModel]] = None, workflow: typing.Optional[airavata.model.workflow.ttypes.AiravataWorkflow] = None,): + def __init__(self, experimentId: str = "DO_NOT_SET_AT_CLIENTS", projectId: str = None, gatewayId: str = None, experimentType: ExperimentType = ExperimentType.SINGLE_APPLICATION, userName: str = None, experimentName: str = None, creationTime: typing.Optional[int] = None, description: typing.Optional[str] = None, executionId: typing.Optional[str] = None, gatewayExecutionId: typing.Optional[str] = None, gatewayInstanceId: typing.Optional[str] = None, enableEmailNotification: typing.Optional[bool] = None, emailAddresses: typing.Optional[list[str]] = None, userConfigurationData: typing.Optional[UserConfigurationDataModel] = None, experimentInputs: typing.Optional[list[airavata.model.application.io.ttypes.InputDataObjectType]] = None, experimentOutputs: typing.Optional[list[airavata.model.application.io.ttypes.OutputDataObjectType]] = None, experimentStatus: typing.Optional[list[airavata.model.status.ttypes.ExperimentStatus]] = None, errors: typing.Optional[list[airavata.model.commons.ttypes.ErrorModel]] = None, processes: typing.Optional[list[airavata.model.process.ttypes.ProcessModel]] = None, workflow: typing.Optional[airavata.model.workflow.ttypes.AiravataWorkflow] = None, cleanUpStrategy: typing.Optional[ExperimentCleanupStrategy] = ExperimentCleanupStrategy.NONE,): self.experimentId: str = experimentId self.projectId: str = projectId self.gatewayId: str = gatewayId @@ -324,11 +333,15 @@ def __init__(self, experimentId: str = "DO_NOT_SET_AT_CLIENTS", projectId: str = self.errors: typing.Optional[list[airavata.model.commons.ttypes.ErrorModel]] = errors self.processes: typing.Optional[list[airavata.model.process.ttypes.ProcessModel]] = processes self.workflow: typing.Optional[airavata.model.workflow.ttypes.AiravataWorkflow] = workflow + self.cleanUpStrategy: typing.Optional[ExperimentCleanupStrategy] = cleanUpStrategy def __setattr__(self, name, value): if name == "experimentType": super().__setattr__(name, value if hasattr(value, 'value') else ExperimentType.__members__.get(value)) return + if name == "cleanUpStrategy": + super().__setattr__(name, value if hasattr(value, 'value') else ExperimentCleanupStrategy.__members__.get(value)) + return super().__setattr__(name, value) @@ -478,6 +491,11 @@ def read(self, iprot): self.workflow.read(iprot) else: iprot.skip(ftype) + elif fid == 21: + if ftype == TType.I32: + self.cleanUpStrategy = ExperimentCleanupStrategy(iprot.readI32()) + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -587,6 +605,10 @@ def write(self, oprot): oprot.writeFieldBegin('workflow', TType.STRUCT, 20) self.workflow.write(oprot) oprot.writeFieldEnd() + if self.cleanUpStrategy is not None: + oprot.writeFieldBegin('cleanUpStrategy', TType.I32, 21) + oprot.writeI32(self.cleanUpStrategy.value) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -1080,6 +1102,7 @@ def __ne__(self, other): (18, TType.LIST, 'errors', (TType.STRUCT, [airavata.model.commons.ttypes.ErrorModel, None], False), None, ), # 18 (19, TType.LIST, 'processes', (TType.STRUCT, [airavata.model.process.ttypes.ProcessModel, None], False), None, ), # 19 (20, TType.STRUCT, 'workflow', [airavata.model.workflow.ttypes.AiravataWorkflow, None], None, ), # 20 + (21, TType.I32, 'cleanUpStrategy', None, ExperimentCleanupStrategy.NONE, ), # 21 ) all_structs.append(ExperimentSummaryModel) ExperimentSummaryModel.thrift_spec = ( From 66c809b47edf57b07c2ac18d9df6b214a791079e Mon Sep 17 00:00:00 2001 From: DImuthuUpe Date: Sat, 29 Nov 2025 10:50:52 -0500 Subject: [PATCH 4/6] Adding test cases to validate the delete directory functionality --- airavata-api/pom.xml | 8 + .../helix/adaptor/SSHJAgentAdaptor.java | 20 +- .../entities/expcatalog/ExperimentEntity.java | 10 +- .../airavata/helix/SFTPDeleteDirTest.java | 173 ++++++++++++++++++ pom.xml | 3 - 5 files changed, 205 insertions(+), 9 deletions(-) create mode 100644 airavata-api/src/test/java/org/apache/airavata/helix/SFTPDeleteDirTest.java diff --git a/airavata-api/pom.xml b/airavata-api/pom.xml index b64be1d8aa..14fe1543ad 100644 --- a/airavata-api/pom.xml +++ b/airavata-api/pom.xml @@ -346,6 +346,14 @@ under the License. + + + org.apache.sshd + sshd-sftp + 2.12.1 + test + + diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java b/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java index 495f9476d4..67161d073c 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java @@ -277,6 +277,24 @@ public void createDirectory(String path, boolean recursive) throws AgentExceptio } } + + private void deleteDirectoryRecursively(SFTPClientWrapper sftpClient, String path) throws IOException { + FileAttributes lstat = sftpClient.lstat(path); + if (lstat.getMode().getType() == Type.DIRECTORY) { + List ls = sftpClient.ls(path); + if (ls == null || ls.isEmpty()) { + sftpClient.rmdir(path); + } else { + for (RemoteResourceInfo r : ls) { + deleteDirectoryRecursively(sftpClient, path + "/" + r.getName()); + } + sftpClient.rmdir(path); + } + } else { + sftpClient.rm(path); + } + } + @Override public void deleteDirectory(String path) throws AgentException { if (path == null || path.trim().isEmpty()) { @@ -285,7 +303,7 @@ public void deleteDirectory(String path) throws AgentException { SFTPClientWrapper sftpClient = null; try { sftpClient = sshjClient.newSFTPClientWrapper(); - sftpClient.rmdir(path); + deleteDirectoryRecursively(sftpClient, path); } catch (Exception e) { if (e instanceof ConnectionException) { Optional.ofNullable(sftpClient).ifPresent(ft -> ft.setErrored(true)); diff --git a/airavata-api/src/main/java/org/apache/airavata/registry/core/entities/expcatalog/ExperimentEntity.java b/airavata-api/src/main/java/org/apache/airavata/registry/core/entities/expcatalog/ExperimentEntity.java index 0a208ecf8c..8055dde100 100644 --- a/airavata-api/src/main/java/org/apache/airavata/registry/core/entities/expcatalog/ExperimentEntity.java +++ b/airavata-api/src/main/java/org/apache/airavata/registry/core/entities/expcatalog/ExperimentEntity.java @@ -50,7 +50,7 @@ public class ExperimentEntity implements Serializable { @Column(name = "CLEANUP_STRATEGY") @Enumerated(EnumType.STRING) - public ExperimentCleanupStrategy cleanupStrategy; + public ExperimentCleanupStrategy cleanUpStrategy; @Column(name = "USER_NAME") public String userName; @@ -269,12 +269,12 @@ public void setExperimentStatus(List experimentStatus) { this.experimentStatus = experimentStatus; } - public ExperimentCleanupStrategy getCleanupStrategy() { - return cleanupStrategy; + public ExperimentCleanupStrategy getCleanUpStrategy() { + return cleanUpStrategy; } - public void setCleanupStrategy(ExperimentCleanupStrategy cleanupStrategy) { - this.cleanupStrategy = cleanupStrategy; + public void setCleanUpStrategy(ExperimentCleanupStrategy cleanUpStrategy) { + this.cleanUpStrategy = cleanUpStrategy; } public List getProcesses() { diff --git a/airavata-api/src/test/java/org/apache/airavata/helix/SFTPDeleteDirTest.java b/airavata-api/src/test/java/org/apache/airavata/helix/SFTPDeleteDirTest.java new file mode 100644 index 0000000000..98bcc626c4 --- /dev/null +++ b/airavata-api/src/test/java/org/apache/airavata/helix/SFTPDeleteDirTest.java @@ -0,0 +1,173 @@ +package org.apache.airavata.helix; + +import org.apache.airavata.helix.adaptor.SSHJAgentAdaptor; +import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory; +import org.apache.sshd.server.SshServer; +import org.apache.sshd.server.config.keys.AuthorizedKeysAuthenticator; +import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider; +import org.apache.sshd.sftp.server.SftpSubsystemFactory; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.List; + +public class SFTPDeleteDirTest { + + private SshServer sshd; + private int port; + private Path sftpRootDir; + private int sftpPort = 52122; + + private String privateKey = "-----BEGIN OPENSSH PRIVATE KEY-----\n" + + "b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABCHHZONdz\n" + + "yrWLbnw4nyEw3BAAAAGAAAAAEAAAGXAAAAB3NzaC1yc2EAAAADAQABAAABgQCxhKaGaUlU\n" + + "Znlr6OtFQ7hjVceZsWLBaIWB5NUwp45IoWLm7Hnor+Y8J/SwLBgdUSsjxkMUQMMbJdY/rP\n" + + "Gwc8aW0K1JMjSNhv03dBxvXHdY+NSd24WjSezD89l6v78lGhVQ5g3rI4eTFsfPy2WSxZw1\n" + + "Fo0UUDVzzBtLuvC9ZCWd3nsT8Ox4LnZWLHrrRxGX2eCotEEO6CT1+wmk4szIkeDDmX79Tb\n" + + "KatcN2vv7H6WjsoGH1bhc1QwS2/hmOdBwqGfm+sE0BI3VgMJ1NVDQrnt0IXlMtLH9Feg1y\n" + + "tdagCzaHulQ9lHn1wBzSARP/NqYzu2vNwpWSSJeafClHpA8yF/9FW9gOi4k+Oo949b+Xd1\n" + + "NHqjt+7lnlsepm0IsgfJ9Gr/0sweYfUSsTfGZGMstSRMu8V+bD1BaVqXQKZ80XoCm0NMnR\n" + + "Chm109wXtt5+0atDmIFiy1Byr8QjwjqsIap1j93R/8R3L3mhUmLruSl7IPKPhjShEIL253\n" + + "GpoHiSENae9e0AAAWgEaos8m239pnUDpWU3N9VtUvg3XVh9WC1YwL9wg1rnl+uW3ygA4Xq\n" + + "VvGUEc5Xx5AR3buKaYGI7+Tb4RAwQL8HkQS78mDtmSiNKJbxmUWkLIWERBe/OZGO/HYPSl\n" + + "WS3nkXogcYy5Q/9Fy4U35Trg82yq/kaSjIneJAGLz0ShbQNgWBtnpzK8eHqceoMFYQsvZ+\n" + + "eaK3JWTwQPgXinj2E37OU5N0y5ncZ8yQ5bKEbOBZ62uYdZFnIgQhz9oNVS8ShIVZtBC0h4\n" + + "ytl45Tdsd4H8cy2RMzzvvLtsfnvA6EOzj5exSNrtsbjZMFvK7f1oatKkm71IknvieGr0nh\n" + + "qvmR+qc15wwnmmFus9MFpqxsOKdPzkeSvBjhe9Oj5Qc9g9ecNHuSuS7MTRcx6UFmB9tvo+\n" + + "iLW0uEzIguQSyaAo1VBNgbr+wz11TaB+rhi2krdUc59skS6/mrah7gJr0kGAJowLR+YGjN\n" + + "/UTJpaEhMWkktuAznY56qs7AlHqKzcNq+258LpIOQJzN9/gw9IB2rz0PNnA+NqDCHttQLw\n" + + "0dZe/oPHJQ6vI/5ykakSas5GJZOph5udSz05ndM5kRoMOGHhi8WeYA0vFBed3BH+lkZ59K\n" + + "z+vjf4sGmOb0ptW95QA9ZeMN899QvuCYOgnuyCPguVL3SsRkQ9AXmOrLT4oPTSUOY3t7vv\n" + + "GI5WN5ZN9zYtT21bOMqYi+cHlIhnaqz+GjRpEfGaqJFPLcj1tVznHbi+2HHCG0M+NTjw9G\n" + + "JRjAjqOfkJZ0/7KmfBT7lGWNPPNgXtYPDdYRHHiIeDMLu4s2gBbqn8pmIdG14K4IqLl7uC\n" + + "payMNJxmQ75oRFpv3Vtf31FlpnsT762iS0e7P0CwBxVZyjdCYet9IVjw6MJC62svnTDznn\n" + + "0ZxPdz78acoXlBkH67zDH69LyPGZlZ9e7HeKrMbOTU5mnUfSiHc3mk8PYEuphnKXFd8Zzi\n" + + "bc/SfaxLbf19MsuqlM+gqKR9hVqDn6Ri9JAmHJBgFNc5hdLSKucunNFFamCslCXRkB3TNl\n" + + "pbPxSLMJ9UDTcrRnzgi5zyQxSe3K8tspqhXQ6ek5Z2sZ+zZuFzcKzgUcd8fpYxC9lZvJ1b\n" + + "pS8OCuGUI6KHHmGJmNKBTbxvp0B4EjRIy3lDJDBMap/GN9GsgqscrvYPIfqlnVR7GXN+qj\n" + + "MgOsue1jtVzG1SBAmBxcctEFLzBsr4k/fNNTXPt/mPKeO3w59zt1OSPyNx63NbNmo/uWO1\n" + + "8P24MBcO5crhlYa5ptb6Fvi1/j6Yrg1NYDPutRopcZNemEFPkR4dqW5AhJwT8L8hqZmmhs\n" + + "DH97qNiqkqyVmrRIygnVMdYqXsn/uV8yEb5mgRw8C6fJ7OZsvwsSfy052tBKJhj/63Ay/S\n" + + "wJ+HxQ/8vthvEkXsaJWiQ2RwatZIoVpOhYEpKwSDuBHMKrnMiCow13+pAq9Gf/CbXUd/Ko\n" + + "xNQ8RZ8lkreUDjJJhTXRRcpufJChL6zQj9bat6E9QBq4l1XjGDhAqgfvQT/1fDataZW3vW\n" + + "skze0s7diqtYIWNlx2+4vGxL38pSCSqtOWjHS6Rbjf37ERKQMH57z4w3aEiahtBcgKTWBy\n" + + "n4UD18TfLGd2i7jtENLxOcWBFzRxtIbFnKGiktLcp0XILs/lOhtRF+K2abiif26rDx++jI\n" + + "4iQCet6ltdeQJLekjmNh4/9Y4hCf5yx9lKuGbzGeZPI66ClbY+R2l29ZXUNUxZmVKM4BDw\n" + + "2LqMlVLcM1Nzg6ftQ09Dku1ApX/uKeOaf0I0DPaBwVD+iTGCeZWuOg5b1LZUuxxYT4ZB6F\n" + + "hoZ8/1mt5gTzo4XdZCmJ7jCOqEc75JV2NEfcIwpy6TOZPVMMWFYT88OgkF86Vxx8GR0FQU\n" + + "CLSDGVZjFU7kv1eKpDJ0oETyGBELC1PPMpm90nxCkzCx7uQw\n" + + "-----END OPENSSH PRIVATE KEY-----\n"; + + private String publicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCxhKaGaUlUZnlr6OtFQ7hjVceZsWLBaIWB5NUwp45IoWLm7Hnor+Y8J/SwLBgdUSsjxkMUQMMbJdY/rPGwc8aW0K1JMjSNhv03dBxvXHdY+NSd24WjSezD89l6v78lGhVQ5g3rI4eTFsfPy2WSxZw1Fo0UUDVzzBtLuvC9ZCWd3nsT8Ox4LnZWLHrrRxGX2eCotEEO6CT1+wmk4szIkeDDmX79TbKatcN2vv7H6WjsoGH1bhc1QwS2/hmOdBwqGfm+sE0BI3VgMJ1NVDQrnt0IXlMtLH9Feg1ytdagCzaHulQ9lHn1wBzSARP/NqYzu2vNwpWSSJeafClHpA8yF/9FW9gOi4k+Oo949b+Xd1NHqjt+7lnlsepm0IsgfJ9Gr/0sweYfUSsTfGZGMstSRMu8V+bD1BaVqXQKZ80XoCm0NMnRChm109wXtt5+0atDmIFiy1Byr8QjwjqsIap1j93R/8R3L3mhUmLruSl7IPKPhjShEIL253GpoHiSENae9e0= dwannipu@Dimuthus-MacBook-Pro.local"; + private String passphrase = "airavata"; + + @BeforeEach + void setUp() throws Exception { + sftpRootDir = Files.createTempDirectory("sftp-root-"); + sftpRootDir.toFile().deleteOnExit(); + + Path authorizedKeysFile = Files.createTempFile("authorized_keys-", ""); + Files.write(authorizedKeysFile, publicKey.getBytes(StandardCharsets.UTF_8)); + authorizedKeysFile.toFile().deleteOnExit(); + + sshd = SshServer.setUpDefaultServer(); + sshd.setHost("localhost"); + sshd.setPort(sftpPort); + + // Host key (for the server itself, unrelated to client auth) + sshd.setKeyPairProvider(new SimpleGeneratorHostKeyProvider()); + + // SFTP subsystem + sshd.setSubsystemFactories( + Collections.singletonList(new SftpSubsystemFactory.Builder().build()) + ); + + // Virtual root + sshd.setFileSystemFactory(new VirtualFileSystemFactory(sftpRootDir)); + + // *** AUTH CONFIG *** + + // 1) Disable password auth (only key-based logins allowed) + sshd.setPasswordAuthenticator(null); + + sshd.setPublickeyAuthenticator(new AuthorizedKeysAuthenticator(authorizedKeysFile)); + + sshd.start(); + port = sshd.getPort(); + } + + @AfterEach + void tearDown() throws Exception { + if (sshd != null && !sshd.isClosed()) { + sshd.stop(true); + } + } + + public void createFilesInDir(Path root) throws IOException { + Path dir1 = Files.createDirectory(root.resolve("dir1")); + dir1.toFile().deleteOnExit(); + Path dir2 = Files.createDirectory(root.resolve("dir2")); + dir2.toFile().deleteOnExit(); + + Path file1 = Files.createFile(root.resolve("file1.txt")); + file1.toFile().deleteOnExit(); + Path file2 = Files.createFile(root.resolve("file2.txt")); + file2.toFile().deleteOnExit(); + + Path file3 = Files.createFile(dir1.resolve("file3.txt")); + file3.toFile().deleteOnExit(); + Path file4 = Files.createFile(dir1.resolve("file4.txt")); + file4.toFile().deleteOnExit(); + + Files.writeString(file1, "Hello from file1\n"); + Files.writeString(file2, "Hello from file2\n"); + Files.writeString(file3, "Hello from file3\n"); + Files.writeString(file4, "Hello from file4\n"); + } + + @Test + public void deleteNonEmptyDir() throws Exception { + System.out.printf("Root dir: %s\n", sftpRootDir); + + Path dir1 = Files.createDirectory(sftpRootDir.resolve("dir1")); + dir1.toFile().deleteOnExit(); + Path dir2 = Files.createDirectory(sftpRootDir.resolve("dir2")); + dir2.toFile().deleteOnExit(); + + createFilesInDir(dir1); + createFilesInDir(dir2); + + SSHJAgentAdaptor adaptor = new SSHJAgentAdaptor(); + adaptor.init("testuser", "localhost", sftpPort, publicKey, privateKey, passphrase); + + List itemsBefore = adaptor.listDirectory("/"); + adaptor.deleteDirectory("dir1"); + List itemsAfter = adaptor.listDirectory("/"); + System.out.printf("Before: %s\n", itemsBefore); + System.out.printf("After: %s\n", itemsAfter); + + } + + @Test + public void deleteEmptyDir() throws Exception { + Path dir1 = Files.createDirectory(sftpRootDir.resolve("dir1")); + dir1.toFile().deleteOnExit(); + + SSHJAgentAdaptor adaptor = new SSHJAgentAdaptor(); + adaptor.init("testuser", "localhost", sftpPort, publicKey, privateKey, passphrase); + List itemsBefore = adaptor.listDirectory("/"); + adaptor.deleteDirectory("dir1"); + List itemsAfter = adaptor.listDirectory("/"); + + Assertions.assertTrue(itemsBefore.get(0).equals("dir1")); + Assertions.assertTrue(itemsAfter.isEmpty()); + } +} diff --git a/pom.xml b/pom.xml index d5cc2be601..8609f5f92e 100644 --- a/pom.xml +++ b/pom.xml @@ -612,9 +612,6 @@ under the License. ${skipTests} ${project.build.testOutputDirectory} false - -Xmx1024m -XX:MaxPermSize=256m --add-opens java.base/java.lang=ALL-UNNAMED - -javaagent:${settings.localRepository}/org/jmockit/jmockit/1.50/jmockit-1.50.jar - false **/IT.java From 398531c48091b831ccbc76fe90cc61b9309312ee Mon Sep 17 00:00:00 2001 From: DImuthuUpe Date: Sat, 29 Nov 2025 14:24:52 -0500 Subject: [PATCH 5/6] Fixed merge conflict and added missing assertions --- .../helix/adaptor/SSHJAgentAdaptor.java | 27 ------------------- .../airavata/helix/SFTPDeleteDirTest.java | 8 ++++-- 2 files changed, 6 insertions(+), 29 deletions(-) diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java b/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java index e4c08bbb59..67161d073c 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java @@ -322,33 +322,6 @@ public void deleteDirectory(String path) throws AgentException { } } - @Override - public void deleteDirectory(String path) throws AgentException { - if (path == null || path.trim().isEmpty()) { - throw new AgentException("Directory path cannot be null or empty"); - } - SFTPClientWrapper sftpClient = null; - try { - sftpClient = sshjClient.newSFTPClientWrapper(); - sftpClient.rmdir(path); - } catch (Exception e) { - if (e instanceof ConnectionException) { - Optional.ofNullable(sftpClient).ifPresent(ft -> ft.setErrored(true)); - } - logger.error("Error while deleting directory {}", path, e); - throw new AgentException(e); - - } finally { - Optional.ofNullable(sftpClient).ifPresent(client -> { - try { - client.close(); - } catch (IOException e) { - // Ignore - } - }); - } - } - @Override public void uploadFile(String localFile, String remoteFile) throws AgentException { SCPFileTransferWrapper fileTransfer = null; diff --git a/airavata-api/src/test/java/org/apache/airavata/helix/SFTPDeleteDirTest.java b/airavata-api/src/test/java/org/apache/airavata/helix/SFTPDeleteDirTest.java index 98bcc626c4..52e67b79d0 100644 --- a/airavata-api/src/test/java/org/apache/airavata/helix/SFTPDeleteDirTest.java +++ b/airavata-api/src/test/java/org/apache/airavata/helix/SFTPDeleteDirTest.java @@ -151,8 +151,12 @@ public void deleteNonEmptyDir() throws Exception { List itemsBefore = adaptor.listDirectory("/"); adaptor.deleteDirectory("dir1"); List itemsAfter = adaptor.listDirectory("/"); - System.out.printf("Before: %s\n", itemsBefore); - System.out.printf("After: %s\n", itemsAfter); + + Assertions.assertTrue(itemsBefore.contains("dir1")); + Assertions.assertTrue(itemsBefore.contains("dir2")); + + Assertions.assertFalse(itemsAfter.contains("dir1")); + Assertions.assertTrue(itemsAfter.contains("dir2")); } From 07d6f313a9ad82eef2a9552dc861d1af1135aa11 Mon Sep 17 00:00:00 2001 From: Dimuthu Wannipurage Date: Sat, 29 Nov 2025 14:25:50 -0500 Subject: [PATCH 6/6] Minor refactoring Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../test/java/org/apache/airavata/helix/SFTPDeleteDirTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airavata-api/src/test/java/org/apache/airavata/helix/SFTPDeleteDirTest.java b/airavata-api/src/test/java/org/apache/airavata/helix/SFTPDeleteDirTest.java index 52e67b79d0..a1de29ecc9 100644 --- a/airavata-api/src/test/java/org/apache/airavata/helix/SFTPDeleteDirTest.java +++ b/airavata-api/src/test/java/org/apache/airavata/helix/SFTPDeleteDirTest.java @@ -171,7 +171,7 @@ public void deleteEmptyDir() throws Exception { adaptor.deleteDirectory("dir1"); List itemsAfter = adaptor.listDirectory("/"); - Assertions.assertTrue(itemsBefore.get(0).equals("dir1")); + Assertions.assertTrue(itemsBefore.contains("dir1")); Assertions.assertTrue(itemsAfter.isEmpty()); } }