devops improvements

[dist] merge assembly XML/content into airavata-api
[api-server] remove redundant dbinit schemas
[api-server] move all templates into templates/
[api-server] move all db scripts into database_scripts/
[devcontainer] start creating full dev environ inside .devcontainer

Author: yasithdev

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM ubuntu:22.04 as deps
 
 ARG TARGETARCH
 
@@ -24,7 +24,7 @@ RUN wget https://go.dev/dl/go1.23.0.linux-amd64.tar.gz \
     && tar -C /usr/local -xzf go1.23.0.linux-amd64.tar.gz \
     && rm go1.23.0.linux-amd64.tar.gz
 
-# Install Thrift 0.23
+# Install Thrift 0.22
 RUN wget https://dlcdn.apache.org/thrift/0.22.0/thrift-0.22.0.tar.gz \
     && tar -xzf thrift-0.22.0.tar.gz \
     && cd thrift-0.22.0 \
@@ -45,3 +45,26 @@ ENV MAVEN_HOME=/opt/maven
 ENV GOROOT=/usr/local/go
 ENV GOPATH=/home/developer/go
 ENV PATH=$PATH:$JAVA_HOME/bin:$MAVEN_HOME/bin:$GOROOT/bin:$GOPATH/bin
+
+EXPOSE 7878 8000 17000 8050 8930 9930 8960 8962 8970 18800 19900 18889 19908
+
+FROM deps AS compiler
+
+COPY --chown=developer:developer .. /home/developer/workspace
+WORKDIR /home/developer/workspace
+RUN mvn clean install -DskipTests
+
+
+FROM eclipse-temurin:17-jdk AS runner
+WORKDIR /app
+
+COPY --from=compiler /home/developer/workspace/distribution/*.tar.gz ./
+
+COPY ../dev-tools/deployment-scripts/*.sh ./
+COPY ../vault ./vault
+
+RUN chmod +x ./distribution_update.sh ./services_up.sh
+
+EXPOSE 7878 8000 17000 8050 8930 9930 8960 8962 8970 18800 19900 18889 19908
+
+CMD ["sh", "-c", "./distribution_update.sh && ./services_up.sh && sleep infinity"]

.devcontainer/database_scripts/init/00-accounts.sql

@@ -0,0 +1,3 @@
+CREATE USER IF NOT EXISTS 'airavata'@'%' IDENTIFIED BY '123456';
+GRANT ALL PRIVILEGES ON *.* TO 'airavata'@'%';
+FLUSH PRIVILEGES;

.devcontainer/database_scripts/init/01-databases.sql

@@ -3519,6 +3519,14 @@ CREATE DATABASE /*!32312 IF NOT EXISTS*/ `workflow_catalog` /*!40100 DEFAULT CHA
 
 USE `workflow_catalog`;
 
+--
+-- Current Database: `research_catalog`
+--
+
+CREATE DATABASE /*!32312 IF NOT EXISTS*/ `research_catalog` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci */;
+
+USE `research_catalog`;
+
 --
 -- Current Database: `app_catalog`
 --

.devcontainer/database_scripts/init/06-cloud-execution-support-migration.sql

@@ -1,3 +1,5 @@
+USE app_catalog;
+
 -- Add RESOURCE_TYPE column to base table
 ALTER TABLE `GROUP_COMPUTE_RESOURCE_PREFERENCE`
     ADD COLUMN `RESOURCE_TYPE` VARCHAR(255) NOT NULL DEFAULT 'SLURM';

.devcontainer/devcontainer.json

@@ -5,7 +5,7 @@
   "dockerComposeFile": "docker-compose.yml",
   "service": "devcontainer",
   "features": {},
-  "workspaceFolder": "/workspace",
+  "workspaceFolder": "/home/developer/workspace",
   "forwardPorts": [],
   "customizations": {
     "vscode": {

.devcontainer/docker-compose.yml

@@ -13,32 +13,114 @@ volumes:
 
 services:
   devcontainer:
-    build: .
+    profiles:
+      - development
+    build:
+      dockerfile: Dockerfile
+      target: deps
     volumes:
-      - ..:/workspace:cached
+      - ..:/home/developer/workspace:cached
+      - $HOME/.m2:/home/developer/.m2
+      - $HOME/.ssh:/home/developer/.ssh
+    networks:
+      airavata-network:
+        ipv4_address: 192.168.100.10
+    extra_hosts:
+      - "airavata.host:192.168.100.1"
+    ports:
+      - "7878:7878"   # sharing registry service
+      - "8000:8000"   # tunnel service
+      - "17000:17000" # tunnel service (ingress)
+      - "8050:8050"   # file service
+      - "8930:8930"   # api service
+      - "9930:9930"   # api service (TLS) !!!
+      - "8960:8960"   # cred store service
+      - "8962:8962"   # profile service
+      - "8970:8970"   # registry service
+      - "18800:18800" # agent service (http)
+      - "19900:19900" # agent service (gRPC)
+      - "18889:18889" # research service (http)
+      - "19908:19908" # research service (gRPC)
     command: sleep infinity
+
+  microservices:
+    profiles:
+      - staging
+    build:
+      dockerfile: Dockerfile
+      target: runner
+    volumes:
+      - ../vault:/app/vault
     networks:
       airavata-network:
         ipv4_address: 192.168.100.10
     extra_hosts:
       - "airavata.host:192.168.100.1"
+    ports:
+      - "7878:7878"   # sharing registry service
+      - "8000:8000"   # tunnel service
+      - "17000:17000" # tunnel service (ingress)
+      - "8050:8050"   # file service
+      - "8930:8930"   # api service
+      - "9930:9930"   # api service (TLS) !!!
+      - "8960:8960"   # cred store service
+      - "8962:8962"   # profile service
+      - "8970:8970"   # registry service
+      - "18800:18800" # agent service (http)
+      - "19900:19900" # agent service (gRPC)
+      - "18889:18889" # research service (http)
+      - "19908:19908" # research service (gRPC)
+    depends_on:
+      - db
+      - kafka
+      - keycloak
+      - rabbitmq
+      - sshd
+
+  portals:
+    build: portals
+    volumes:
+      - /tmp:/tmp
+      - ./portals/pga_config.php:/var/www/html/default/config/pga_config.php
+    networks:
+      airavata-network:
+        ipv4_address: 192.168.100.18
+    extra_hosts:
+      - "airavata.host:192.168.100.1"
+
+  proxy:
+    image: nginx/nginx-ingress
+    ports:
+      - "8443:8443" # https keycloak
+      - "5173:5173" # https research portal
+      - "8008:8008" # https php portal
+      - "8009:8009" # https django portal
+    environment:
+      RESEARCH_PORTAL_ADDR: "portals:5173"
+      PHP_PORTAL_ADDR: "portals:8008"
+      DJANGO_PORTAL_ADDR: "portals:8009"
+      KEYCLOAK_ADDR: "keycloak:18080"
+    volumes:
+      - ../vault:/vault  # requires /vault/server.crt and /vault/server.key for ssl termination
+      - ./proxy/proxy.conf:/etc/nginx/conf.d/proxy.conf:ro
+    networks:
+      airavata-network:
+        ipv4_address: 192.168.100.18
+    extra_hosts:
+      - "airavata.host:192.168.100.1"
 
   keycloak:
-    image: keycloak/keycloak:24.0
+    image: keycloak/keycloak:25.0
     restart: always
     environment:
       KEYCLOAK_ADMIN: admin
       KEYCLOAK_ADMIN_PASSWORD: admin
     ports:
-      - "18080:8080"
-      - "8443:8443"
+      - "18080:18080"
     volumes:
       - ./keycloak/realm-default.json:/opt/keycloak/data/import/realm-default.json
       - ./keycloak/keycloak.conf:/opt/keycloak/conf/keycloak.conf
-      - ../keystores/airavata.jks:/opt/keycloak/conf/keystores/airavata.jks
-    command: ["start-dev", "--import-realm"]
-    depends_on:
-      - db
+    command: [ "start", "--import-realm" ]
     networks:
       airavata-network:
         ipv4_address: 192.168.100.11
@@ -78,8 +160,6 @@ services:
     ports:
       - "5672:5672"
       - "15672:15672"
-    depends_on:
-      - db
     networks:
       airavata-network:
         ipv4_address: 192.168.100.13
@@ -91,7 +171,7 @@ services:
     restart: always
     hostname: zk
     ports:
-      - "12181:2181"
+      - "2181:2181"
     networks:
       airavata-network:
         ipv4_address: 192.168.100.14
@@ -115,8 +195,26 @@ services:
     extra_hosts:
       - "airavata.host:192.168.100.1"
 
+  kafka-rest:
+    image: confluentinc/cp-kafka-rest:latest
+    restart: always
+    depends_on:
+      - kafka
+    ports:
+      - "8082:8082"
+    environment:
+      KAFKA_REST_HOST_NAME: 192.168.100.19
+      KAFKA_REST_LISTENERS: http://0.0.0.0:8082
+      KAFKA_REST_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9092
+      KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper:2181
+    networks:
+      airavata-network:
+        ipv4_address: 192.168.100.19
+    extra_hosts:
+      - "airavata.host:192.168.100.1"
+
   sshd:
-    image: dimuthuupe/sshd:1.0
+    image: panubo/sshd
     restart: always
     volumes:
       - /tmp:/tmp
@@ -140,15 +238,3 @@ services:
         ipv4_address: 192.168.100.17
     extra_hosts:
       - "airavata.host:192.168.100.1"
-
-  pga:
-    build: pga
-    volumes:
-      - /tmp:/tmp
-    ports:
-      - "8000:80"
-    networks:
-      airavata-network:
-        ipv4_address: 192.168.100.18
-    extra_hosts:
-      - "airavata.host:192.168.100.1"

.devcontainer/keycloak/keycloak.conf

@@ -1,20 +1,9 @@
-# Keycloak Configuration File
 db=dev-mem
-
-# HTTP/HTTPS settings
 http-enabled=true
-http-port=8080
-https-port=8443
-https-key-store-file=${kc.home.dir}/conf/keystores/airavata.jks
-https-key-store-password=airavata
-
-# Hostname configuration
+http-port=18080
+https-port=0
 hostname-strict=false
 hostname-strict-https=false
-
-# Health and metrics
 health-enabled=true
 metrics-enabled=true
-
-# Logging
 log-level=INFO

.devcontainer/keycloak/realm-default.json

@@ -957,7 +957,7 @@
     "alwaysDisplayInConsole" : false,
     "clientAuthenticatorType" : "client-secret",
     "secret" : "DxeMtfiWU1qkDEmaGHf13RDahCujzhy1",
-    "redirectUris" : [ "", "/*", "https://hub.cybershuttle.org/hub/oauth_callback" ],
+    "redirectUris" : [ "", "/*", "http://airavata.host:20000/hub/oauth_callback" ],
     "webOrigins" : [ "/*" ],
     "notBefore" : 0,
     "bearerOnly" : false,
@@ -1052,7 +1052,7 @@
     "description" : "",
     "rootUrl" : "",
     "adminUrl" : "",
-    "baseUrl" : "https://cybershuttle.org/",
+    "baseUrl" : "http://airavata.host:8009/",
     "surrogateAuthRequired" : false,
     "enabled" : true,
     "alwaysDisplayInConsole" : false,
@@ -1102,8 +1102,8 @@
     "alwaysDisplayInConsole" : false,
     "clientAuthenticatorType" : "client-secret",
     "secret" : "m36BXQIxX3j3VILadeHMK5IvbOeRlCCc",
-    "redirectUris" : [ "https://cybershuttle.org/callback-url", "http://localhost/callback-url", "http://airavata.host:8008/callback-url", "https://cybershuttle.org/", "https://cybershuttle.org/auth/callback*" ],
-    "webOrigins" : [ "https://cybershuttle.org", "*" ],
+    "redirectUris" : [ "http://airavata.host:8008/callback*", "https://airavata.host:8009/auth/callback*" ],
+    "webOrigins" : [ "*" ],
     "notBefore" : 0,
     "bearerOnly" : false,
     "consentRequired" : false,
@@ -1119,8 +1119,8 @@
       "oidc.ciba.grant.enabled" : "false",
       "client.secret.creation.time" : "1741724922",
       "backchannel.logout.session.required" : "true",
-      "frontchannel.logout.url" : "https://cybershuttle.org/",
-      "post.logout.redirect.uris" : "+##https://cybershuttle.org/",
+      "frontchannel.logout.url" : "http://airavata.host:8009/",
+      "post.logout.redirect.uris" : "+##http://airavata.host:8009/",
       "display.on.consent.screen" : "false",
       "oauth2.device.authorization.grant.enabled" : "true",
       "backchannel.logout.revoke.offline.tokens" : "false"
@@ -1922,7 +1922,7 @@
       "forwardParameters" : "kc_idp_hint",
       "authorizationUrl" : "https://cilogon.org/authorize",
       "disableUserInfo" : "false",
-      "logoutUrl" : "https://cybershuttle.org/",
+      "logoutUrl" : "http://airavata.host:8009/",
       "sendIdTokenOnLogout" : "true",
       "passMaxAge" : "false"
     }