Support consecutive batch job runs in agent

.devcontainer/database_scripts/init/07-parameter-sweep-modeling.sql

@@ -0,0 +1,45 @@
+USE `app_catalog`;
+
+-- Parent collection for one parameter sweep / batch
+CREATE TABLE IF NOT EXISTS `JOB_BATCH`
+(
+    `ID`               VARCHAR(255) NOT NULL,
+    `EXPERIMENT_ID`    VARCHAR(255) NOT NULL,
+    `CREATED_AT`       TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+    `PAYLOAD_JSON`     JSON         NULL,     -- original request payload
+    `COMMAND_TEMPLATE` TEXT         NOT NULL, -- application_command (template)
+    PRIMARY KEY (`ID`),
+    KEY `IDX_BATCH_EXPERIMENT` (`EXPERIMENT_ID`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+-- Units of work (one per parameter combination)
+CREATE TABLE IF NOT EXISTS `JOB_UNIT`
+(
+    `ID`               VARCHAR(255)                                        NOT NULL,
+    `BATCH_ID`         VARCHAR(255)                                        NOT NULL,
+    `EXPERIMENT_ID`    VARCHAR(255)                                        NOT NULL,
+    `CREATED_AT`       TIMESTAMP(6)                                        NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+    `RESOLVED_COMMAND` TEXT                                                NOT NULL, -- fully expanded command to run
+    `STATUS` ENUM ('PENDING','IN_PROGRESS','COMPLETED','FAILED') NOT NULL DEFAULT 'PENDING',
+    `AGENT_ID`         VARCHAR(255)                                        NULL,
+    `STARTED_AT`       TIMESTAMP(6)                                        NULL,
+    `COMPLETED_AT`     TIMESTAMP(6)                                        NULL,
+    PRIMARY KEY (`ID`),
+    KEY `IDX_UNIT_BATCH_STATUS_FIFO` (`BATCH_ID`, `STATUS`, `CREATED_AT`, `ID`),
+    KEY `IDX_UNIT_EXP_STATUS` (`EXPERIMENT_ID`, `STATUS`),
+    CONSTRAINT `FK_JOB_UNIT_BATCH` FOREIGN KEY (`BATCH_ID`)
+        REFERENCES `JOB_BATCH` (`ID`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+-- Agent Batch Job assignment
+CREATE TABLE IF NOT EXISTS `AGENT_BATCH_ASSIGNMENT`
+(
+    `AGENT_ID`      VARCHAR(255) NOT NULL,
+    `EXPERIMENT_ID` VARCHAR(255) NOT NULL,
+    `BATCH_ID`      VARCHAR(255) NOT NULL,
+    `CREATED_AT`    TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+    PRIMARY KEY (`AGENT_ID`),
+    KEY `IDX_ASSIGN_BATCH` (`BATCH_ID`),
+    CONSTRAINT `FK_ASSIGN_BATCH` FOREIGN KEY (`BATCH_ID`)
+        REFERENCES `JOB_BATCH` (`ID`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

modules/agent-framework/agent-service/src/main/java/org/apache/airavata/agent/connection/service/config/AsyncConfig.java

@@ -0,0 +1,41 @@
+/**
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+package org.apache.airavata.agent.connection.service.config;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+
+@EnableAsync
+@Configuration
+public class AsyncConfig {
+
+    @Bean(name = "batchExecutor")
+    public ThreadPoolTaskExecutor batchExecutor() {
+        ThreadPoolTaskExecutor ex = new ThreadPoolTaskExecutor();
+        ex.setThreadNamePrefix("batch-");
+        ex.setCorePoolSize(4);
+        ex.setMaxPoolSize(8);
+        ex.setQueueCapacity(1000);
+        ex.initialize();
+        return ex;
+    }
+}

modules/agent-framework/agent-service/src/main/java/org/apache/airavata/agent/connection/service/db/entity/AgentBatchAssignmentEntity.java

@@ -0,0 +1,76 @@
+/**
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+package org.apache.airavata.agent.connection.service.db.entity;
+
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.Id;
+import jakarta.persistence.Table;
+import java.time.Instant;
+
+@Entity
+@Table(name = "AGENT_BATCH_ASSIGNMENT")
+public class AgentBatchAssignmentEntity {
+
+    @Id
+    @Column(name = "AGENT_ID", nullable = false)
+    private String agentId;
+
+    @Column(name = "EXPERIMENT_ID", nullable = false)
+    private String experimentId;
+
+    @Column(name = "BATCH_ID", nullable = false)
+    private String batchId;
+
+    @Column(name = "CREATED_AT", insertable = false, updatable = false)
+    private Instant createdAt;
+
+    public String getAgentId() {
+        return agentId;
+    }
+
+    public void setAgentId(String agentId) {
+        this.agentId = agentId;
+    }
+
+    public String getExperimentId() {
+        return experimentId;
+    }
+
+    public void setExperimentId(String experimentId) {
+        this.experimentId = experimentId;
+    }
+
+    public String getBatchId() {
+        return batchId;
+    }
+
+    public void setBatchId(String batchId) {
+        this.batchId = batchId;
+    }
+
+    public Instant getCreatedAt() {
+        return createdAt;
+    }
+
+    public void setCreatedAt(Instant createdAt) {
+        this.createdAt = createdAt;
+    }
+}

modules/agent-framework/agent-service/src/main/java/org/apache/airavata/agent/connection/service/db/entity/JobBatchEntity.java

@@ -0,0 +1,108 @@
+/**
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+package org.apache.airavata.agent.connection.service.db.entity;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import jakarta.persistence.CascadeType;
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.FetchType;
+import jakarta.persistence.Id;
+import jakarta.persistence.Lob;
+import jakarta.persistence.OneToMany;
+import jakarta.persistence.Table;
+import java.time.Instant;
+import java.util.List;
+import org.hibernate.annotations.JdbcTypeCode;
+import org.hibernate.type.SqlTypes;
+
+@Entity
+@Table(name = "JOB_BATCH")
+public class JobBatchEntity {
+
+    @Id
+    @Column(name = "ID", nullable = false)
+    private String id;
+
+    @Column(name = "EXPERIMENT_ID", nullable = false)
+    private String experimentId;
+
+    @Column(name = "CREATED_AT", updatable = false, insertable = false)
+    private Instant createdAt;
+
+    @JdbcTypeCode(SqlTypes.JSON)
+    @Column(name = "PAYLOAD_JSON", columnDefinition = "json")
+    private JsonNode payloadJson;
+
+    @Lob
+    @Column(name = "COMMAND_TEMPLATE", nullable = false)
+    private String commandTemplate;
+
+    @OneToMany(mappedBy = "batch", fetch = FetchType.LAZY, cascade = CascadeType.REMOVE)
+    private List<JobUnitEntity> units;
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getExperimentId() {
+        return experimentId;
+    }
+
+    public void setExperimentId(String experimentId) {
+        this.experimentId = experimentId;
+    }
+
+    public Instant getCreatedAt() {
+        return createdAt;
+    }
+
+    public void setCreatedAt(Instant createdAt) {
+        this.createdAt = createdAt;
+    }
+
+    public JsonNode getPayloadJson() {
+        return payloadJson;
+    }
+
+    public void setPayloadJson(JsonNode payloadJson) {
+        this.payloadJson = payloadJson;
+    }
+
+    public String getCommandTemplate() {
+        return commandTemplate;
+    }
+
+    public void setCommandTemplate(String commandTemplate) {
+        this.commandTemplate = commandTemplate;
+    }
+
+    public List<JobUnitEntity> getUnits() {
+        return units;
+    }
+
+    public void setUnits(List<JobUnitEntity> units) {
+        this.units = units;
+    }
+}