diff --git a/airavata-api/src/main/java/org/apache/airavata/registry/core/repositories/appcatalog/GwyResourceProfileRepository.java b/airavata-api/src/main/java/org/apache/airavata/registry/core/repositories/appcatalog/GwyResourceProfileRepository.java index b5aaf0b245..13029d248a 100644 --- a/airavata-api/src/main/java/org/apache/airavata/registry/core/repositories/appcatalog/GwyResourceProfileRepository.java +++ b/airavata-api/src/main/java/org/apache/airavata/registry/core/repositories/appcatalog/GwyResourceProfileRepository.java @@ -63,6 +63,8 @@ public String updateGatewayResourceProfile(GatewayResourceProfile gatewayResourc String gatewayId = gatewayResourceProfile.getGatewayID(); Mapper mapper = ObjectMapperSingleton.getInstance(); GatewayProfileEntity gatewayProfileEntity = mapper.map(gatewayResourceProfile, GatewayProfileEntity.class); + // Explicitly set gatewayId since Dozer mapping does not handle gatewayID -> gatewayId conversion + gatewayProfileEntity.setGatewayId(gatewayId); if (get(gatewayId) != null) { gatewayProfileEntity.setUpdateTime(AiravataUtils.getCurrentTimestamp()); } else { diff --git a/airavata-api/src/main/resources/database_scripts/appcatalog-mysql.sql b/airavata-api/src/main/resources/database_scripts/appcatalog-mysql.sql index 0d960e6541..28a3bfb12f 100644 --- a/airavata-api/src/main/resources/database_scripts/appcatalog-mysql.sql +++ b/airavata-api/src/main/resources/database_scripts/appcatalog-mysql.sql @@ -592,26 +592,50 @@ CREATE TABLE GROUP_COMPUTE_RESOURCE_PREFERENCE ( RESOURCE_ID VARCHAR(255) NOT NULL, GROUP_RESOURCE_PROFILE_ID varchar(255) NOT NULL, + RESOURCE_TYPE VARCHAR(255) NOT NULL, OVERRIDE_BY_AIRAVATA SMALLINT, PREFERED_JOB_SUB_PROTOCOL VARCHAR(255), PREFERED_DATA_MOVE_PROTOCOL VARCHAR(255), - PREFERED_BATCH_QUEUE VARCHAR(255), SCRATCH_LOCATION VARCHAR(255), - ALLOCATION_PROJECT_NUMBER VARCHAR(255), LOGIN_USERNAME VARCHAR(255), RESOURCE_CS_TOKEN VARCHAR(255), - USAGE_REPORTING_GATEWAY_ID VARCHAR(255), - QUALITY_OF_SERVICE VARCHAR(255), - RESERVATION VARCHAR (255), - RESERVATION_START_TIME timestamp, - RESERVATION_END_TIME timestamp, - SSH_ACCOUNT_PROVISIONER VARCHAR(255), - SSH_ACCOUNT_PROVISIONER_ADDITIONAL_INFO VARCHAR(1000), PRIMARY KEY(RESOURCE_ID,GROUP_RESOURCE_PROFILE_ID), FOREIGN KEY (RESOURCE_ID) REFERENCES COMPUTE_RESOURCE(RESOURCE_ID) ON DELETE CASCADE, FOREIGN KEY (GROUP_RESOURCE_PROFILE_ID) REFERENCES GROUP_RESOURCE_PROFILE(GROUP_RESOURCE_PROFILE_ID) ON DELETE CASCADE )ENGINE=InnoDB DEFAULT CHARSET=latin1; +CREATE TABLE SLURM_GROUP_COMPUTE_RESOURCE_PREFERENCE +( + RESOURCE_ID VARCHAR(255) NOT NULL, + GROUP_RESOURCE_PROFILE_ID VARCHAR(255) NOT NULL, + PREFERED_BATCH_QUEUE VARCHAR(255) DEFAULT NULL, + ALLOCATION_PROJECT_NUMBER VARCHAR(255) DEFAULT NULL, + USAGE_REPORTING_GATEWAY_ID VARCHAR(255) DEFAULT NULL, + QUALITY_OF_SERVICE VARCHAR(255) DEFAULT NULL, + RESERVATION VARCHAR(255) DEFAULT NULL, + RESERVATION_START_TIME TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + RESERVATION_END_TIME TIMESTAMP NOT NULL DEFAULT '0000-00-00 00:00:00', + SSH_ACCOUNT_PROVISIONER VARCHAR(255) DEFAULT NULL, + SSH_ACCOUNT_PROVISIONER_ADDITIONAL_INFO TEXT DEFAULT NULL, + PRIMARY KEY (RESOURCE_ID, GROUP_RESOURCE_PROFILE_ID), + CONSTRAINT FK_SLURM_PREF_TO_BASE FOREIGN KEY (RESOURCE_ID, GROUP_RESOURCE_PROFILE_ID) + REFERENCES GROUP_COMPUTE_RESOURCE_PREFERENCE (RESOURCE_ID, GROUP_RESOURCE_PROFILE_ID) + ON DELETE CASCADE +)ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE AWS_GROUP_COMPUTE_RESOURCE_PREFERENCE +( + RESOURCE_ID VARCHAR(255) NOT NULL, + GROUP_RESOURCE_PROFILE_ID VARCHAR(255) NOT NULL, + AWS_REGION VARCHAR(255) NOT NULL, + PREFERRED_AMI_ID VARCHAR(255) NOT NULL, + PREFERRED_INSTANCE_TYPE VARCHAR(255) NOT NULL, + PRIMARY KEY (RESOURCE_ID, GROUP_RESOURCE_PROFILE_ID), + CONSTRAINT FK_AWS_PREF_TO_BASE FOREIGN KEY (RESOURCE_ID, GROUP_RESOURCE_PROFILE_ID) + REFERENCES GROUP_COMPUTE_RESOURCE_PREFERENCE (RESOURCE_ID, GROUP_RESOURCE_PROFILE_ID) + ON DELETE CASCADE +)ENGINE=InnoDB DEFAULT CHARSET=latin1; + CREATE TABLE COMPUTE_RESOURCE_RESERVATION -- ComputeResourceReservationEntity (RESERVATION_ID VARCHAR(255) NOT NULL, END_TIME TIMESTAMP NOT NULL, RESERVATION_NAME VARCHAR(255) NOT NULL, START_TIME TIMESTAMP NOT NULL, RESOURCE_ID VARCHAR(255) NOT NULL, GROUP_RESOURCE_PROFILE_ID VARCHAR(255) NOT NULL, PRIMARY KEY (RESERVATION_ID) )ENGINE=InnoDB DEFAULT CHARSET=latin1; diff --git a/dev-tools/ansible/DEVELOPER_GUIDE.md b/dev-tools/ansible/DEVELOPER_GUIDE.md new file mode 100644 index 0000000000..93b8358c3c --- /dev/null +++ b/dev-tools/ansible/DEVELOPER_GUIDE.md @@ -0,0 +1,350 @@ +# Airavata Deployment - Developer Quick Guide + +## Prerequisites + +- Ansible installed (or use `venv` in `dev-tools/ansible/`) +- SSH access to target server with sudo privileges +- Vault password (if using encrypted vault files) + +## Setting Up a New Environment + +### Step 1: Copy Template + +```bash +cd dev-tools/ansible +cp -r inventories/template inventories/my-env +cd inventories/my-env +``` + +### Step 2: Rename Example Files + +```bash +mv hosts.example hosts +mv group_vars/all/vars.yml.example group_vars/all/vars.yml +mv group_vars/all/vault.yml.example group_vars/all/vault.yml +mv host_vars/airavata-server/vault.yml.example host_vars/airavata-server/vault.yml +``` + +### Step 3: Edit Configuration Files + +**Edit `hosts`** +- Replace `airavata-server` with your host alias if needed + +**Edit `group_vars/all/vars.yml` (Non-sensitive):** +- Set deployment user, ports, paths +- Set git branch, version +- Replace any `CHANGEME` values + +**Edit `group_vars/all/vault.yml` (Sensitive):** +- Replace all `CHANGEME_*` values: + - Database passwords (`CHANGEME_DB_PASSWORD`) + - Database host (`CHANGEME_DB_HOST` in JDBC URLs) + - IAM credentials (`CHANGEME_IAM_PASSWORD`) + - Keycloak admin password (`CHANGEME_KEYCLOAK_ADMIN_PASSWORD`) + - Keycloak database password (`CHANGEME_KEYCLOAK_DB_PASSWORD`) + - Keycloak client secrets: + - `CHANGEME_PGA_CLIENT_SECRET` - PGA (Gateway) client secret + - `CHANGEME_JUPYTERLAB_CLIENT_SECRET` - JupyterLab client secret + - `CHANGEME_CILOGON_CLIENT_SECRET` - CILogon identity provider secret + - Keycloak client IDs: + - `CHANGEME_CILOGON_CLIENT_ID` - CILogon client ID + - Keycloak redirect URIs (update `CHANGEME_GATEWAY_HOST` and `CHANGEME_JUPYTERLAB_HOST`) + - OAuth secrets (`CHANGEME_OAUTH_SECRET`) + - Keystore passwords (`CHANGEME_KEYSTORE_PASSWORD`) + - Email passwords (`CHANGEME_EMAIL_PASSWORD`) + - Tunnel tokens (`CHANGEME_TUNNEL_TOKEN`) + +**Edit `host_vars/airavata-server/vault.yml` (Sensitive):** +- Set `ansible_host`: Server IP address +- Set `ansible_user`: SSH user for deployment + +### Step 4: Encrypt Vault Files + +```bash +# Encrypt group variables (database passwords, API keys, etc.) +ansible-vault encrypt group_vars/all/vault.yml + +# Encrypt host variables (server IPs, SSH credentials) +ansible-vault encrypt host_vars/airavata-server/vault.yml +``` + +**Note:** You'll be prompted to set a vault password. Remember this password - you'll need it for all playbook runs. + +### Step 5: Test Connection + +```bash +cd ../.. +ansible -i inventories/my-env airavata_servers -m ping --ask-vault-pass +``` + +## Quick Start + +### 1. Initial Setup (First Time) + +```bash +cd dev-tools/ansible + +# Activate virtual environment +source venv/bin/activate + +# Run full setup +ansible-playbook -i inventories/my-env airavata_setup.yml --ask-vault-pass +``` + +**What this does:** +- Creates `airavata` user/group +- Installs Java, Maven, Git +- Sets up Zookeeper, Kafka, RabbitMQ, MariaDB +- Installs Keycloak (IAM) +- Builds and deploys all Airavata services +- Starts all services + +### 2. Update Existing Deployment + +```bash +cd dev-tools/ansible +source venv/bin/activate + +# Update services (stops, builds, deploys, starts) +ansible-playbook -i inventories/my-env airavata_update.yml --ask-vault-pass +``` + +## Vault File Management + +### Encrypt Vault Files + +```bash +cd dev-tools/ansible + +# Encrypt vault files (will prompt for password) +ansible-vault encrypt inventories/my-env/group_vars/all/vault.yml +ansible-vault encrypt inventories/my-env/host_vars/dev-server/vault.yml +``` + +**Important:** Set a strong password and store it securely. You'll need it every time you run playbooks. + +### Decrypt Vault Files + +```bash +ansible-vault decrypt inventories/my-env/group_vars/all/vault.yml +``` + +**Note:** Decrypting removes encryption. Re-encrypt after editing. + +### Edit Encrypted Vault (Recommended) + +```bash +# Opens editor, decrypts temporarily, re-encrypts on save +ansible-vault edit inventories/my-env/group_vars/all/vault.yml +``` + +**Best Practice:** Use `ansible-vault edit` instead of decrypt/edit/encrypt to avoid leaving unencrypted files. + +### View Encrypted Vault (Without Decrypting) + +```bash +ansible-vault view inventories/my-env/group_vars/all/vault.yml +``` + +### Change Vault Password + +```bash +ansible-vault rekey inventories/my-env/group_vars/all/vault.yml +# Enter old password, then new password +``` + +## Service Management + +### Stop All Services + +```bash +cd dev-tools/ansible +ansible-playbook -i inventories/my-env stop_services.yml --ask-vault-pass +``` + +**What this does:** +- Stops all Airavata services gracefully +- Checks for processes holding ports +- Kills processes if needed (SIGTERM, then SIGKILL) +- Verifies all ports are free + +### Start All Services + +```bash +cd dev-tools/ansible +ansible-playbook -i inventories/my-env start_services.yml --ask-vault-pass +``` + +**What this does:** +- Starts all Airavata services +- Waits for services to be ready +- Verifies ports are listening + +## Configuration Files + +### Key Files to Edit + +1. **`inventories/my-env/group_vars/all/vars.yml`** (Non-sensitive) + - Deployment user, ports, paths + - Git branch, version + +2. **`inventories/my-env/group_vars/all/vault.yml`** (Sensitive - encrypted) + - Database passwords, URLs + - IAM/Keycloak credentials (admin password, database password) + - Keycloak realm client configuration (PGA, JupyterLab, CILogon secrets and redirect URIs) + - OAuth secrets + - Keystore passwords + +3. **`inventories/my-env/host_vars/airavata-server/vault.yml`** (Sensitive - encrypted) + - Server connection details + - `ansible_host`, `ansible_user` + +### Important Variables + +| Variable | File | Description | +|---------|------|-------------| +| `deploy_user` | `vars.yml` | User for deployment (`airavata`) | +| `user`, `group` | `vars.yml` | System user/group (`airavata`) | +| `db_password` | `vault.yml` | Database password for all services | +| `iam_server_url` | `vault.yml` | Keycloak URL | +| `keycloak_master_account_password` | `vault.yml` | Keycloak admin password | +| `keycloak_db_password` | `vault.yml` | Keycloak database password | +| `keycloak_pga_client_secret` | `vault.yml` | PGA (Gateway) OAuth client secret | +| `keycloak_pga_redirect_uris` | `vault.yml` | PGA redirect URIs (list) | +| `keycloak_jupyterlab_client_secret` | `vault.yml` | JupyterLab OAuth client secret | +| `keycloak_cilogon_client_id` | `vault.yml` | CILogon identity provider client ID | +| `keycloak_cilogon_client_secret` | `vault.yml` | CILogon identity provider secret | +| `rabbitmq_broker_url` | `vault.yml` | RabbitMQ connection string | +| `*_jdbc_url` | `vault.yml` | Database URLs (use `localhost` or DB server IP) | + +### Keystore Management + +**Automatic Generation (Default):** +The keystore file (`airavata.sym.p12`) is automatically generated from Let's Encrypt certificates during deployment. No manual action needed. + +**Manual Keystore (Optional):** +If you need to use a custom keystore file: +1. Place it in `inventories/my-env/files/airavata.sym.p12` +2. Optionally encrypt it: `ansible-vault encrypt inventories/my-env/files/airavata.sym.p12` +3. The playbook will use your file instead of auto-generating + +## Common Workflows + +### Fresh Deployment + +```bash +# 1. Configure inventory files +# - Edit inventories/my-env/group_vars/all/vars.yml +# - Edit inventories/my-env/group_vars/all/vault.yml (decrypt first if encrypted) + +# 2. Encrypt vault files +ansible-vault encrypt inventories/my-env/group_vars/all/vault.yml +ansible-vault encrypt inventories/my-env/host_vars/airavata-server/vault.yml + +# 3. Run setup +ansible-playbook -i inventories/my-env airavata_setup.yml --ask-vault-pass +``` + +### Update Code and Redeploy + +```bash +# 1. Update code (if needed, change git_branch in vars.yml) + +# 2. Run update (stops, builds, deploys, starts) +ansible-playbook -i inventories/my-env airavata_update.yml --ask-vault-pass +``` + +### Change Configuration + +```bash +# 1. Edit encrypted vault +ansible-vault edit inventories/my-env/group_vars/all/vault.yml + +# 2. Stop services +ansible-playbook -i inventories/my-env stop_services.yml --ask-vault-pass + +# 3. Update deployment (will regenerate configs) +ansible-playbook -i inventories/my-env airavata_update.yml --ask-vault-pass --skip-tags build +``` + +### Restart Services Only + +```bash +# Stop +ansible-playbook -i inventories/my-env stop_services.yml --ask-vault-pass + +# Start +ansible-playbook -i inventories/my-env start_services.yml --ask-vault-pass +``` + +## Verification + +### Check Services Are Running + +```bash +# Check ports +ansible airavata-server -i inventories/my-env -m shell \ + -a "ss -tuln | grep -E '8930|8940|8962|8970|8960|7878|18880|18899|8050|8082'" + +# Check processes +ansible airavata-server -i inventories/my-env -m shell \ + -a "ps aux | grep java | grep airavata | wc -l" +``` + +### Check Service Logs + +```bash +# SSH to server +ssh @ + +# View logs +tail -f /home/airavata//apache-airavata-api-server-*/logs/*.log +``` + +### Check Third-Party Services + +```bash +ansible airavata-server -i inventories/my-env -m shell \ + -a "systemctl status zookeeper kafka rabbitmq-server mariadb keycloak" +``` + +## Troubleshooting + +### Services Won't Start + +1. **Check ports are free:** + ```bash + ansible-playbook -i inventories/my-env stop_services.yml --ask-vault-pass + ``` + +2. **Check database connection:** + - Verify `*_jdbc_url` in vault.yml point to correct database + - Test: `mysql -h -u -p` + +3. **Check RabbitMQ:** + - Verify vhost exists: `rabbitmqctl list_vhosts` + - Verify user has permissions + +4. **Check logs:** + ```bash + ssh + tail -f /home/airavata//apache-airavata-api-server-*/logs/*.log + ``` + +## Quick Reference + +| Task | Command | +|------|---------| +| Full setup | `ansible-playbook -i inventories/my-env airavata_setup.yml --ask-vault-pass` | +| Update services | `ansible-playbook -i inventories/my-env airavata_update.yml --ask-vault-pass` | +| Stop services | `ansible-playbook -i inventories/my-env stop_services.yml --ask-vault-pass` | +| Start services | `ansible-playbook -i inventories/my-env start_services.yml --ask-vault-pass` | +| Encrypt vault | `ansible-vault encrypt inventories/my-env/group_vars/all/vault.yml` | +| Edit vault | `ansible-vault edit inventories/my-env/group_vars/all/vault.yml` | +| Decrypt vault | `ansible-vault decrypt inventories/my-env/group_vars/all/vault.yml` | + +## Additional Resources + +- **`inventories/template/README.md`** - Detailed template setup guide +- **`SETUP_FLOW.md`** - Detailed role descriptions and multi-host configuration +- **`inventories/template/group_vars/all/vault.yml.example`** - Template with all `CHANGEME` placeholders \ No newline at end of file diff --git a/dev-tools/ansible/README.md b/dev-tools/ansible/README.md index 1f94a505c1..78f969686a 100644 --- a/dev-tools/ansible/README.md +++ b/dev-tools/ansible/README.md @@ -1,66 +1,187 @@ -# airavata-ansible +# Apache Airavata Ansible Deployment -Ansible script to deploy Apache Airavata and the Airavata Django Portal. There -are ansible roles to install Airavata pre-requisites (RabbitMQ, Zookeeper, -MariaDB). +Ansible playbooks for deploying and managing Apache Airavata and its dependencies. -## Ansible installation +## Quick Start -Note: the following assumes a Bash shell. +### Prerequisites -1. Download and install the latest version of Python. Minimum required version - is 3.8. See https://www.python.org/downloads/ or use your system's package - manager. -2. Create a virtual environment in this directory +- Python 3.8+ +- SSH access to target servers with sudo privileges +- DNS configured (for Let's Encrypt SSL certificates) - cd airavata/dev-tools/ansible - python3 -m venv ENV +### Installation -3. Source the environment (you'll need to do this each time before using - ansible commands) +```bash +cd dev-tools/ansible - source ENV/bin/activate +# Create virtual environment +python3 -m venv venv +source venv/bin/activate -4. Install ansible and any other dependencies. +# Install dependencies +pip install -r requirements.txt +``` - pip install -r requirements.txt +## Main Playbooks -Now you should be ready to run `ansible-playbook` and other ansible commands. +| Playbook | Purpose | +|----------|---------| +| `airavata_setup.yml` | Complete initial setup from scratch | +| `airavata_update.yml` | Update existing deployment (rebuilds and redeploys) | +| `start_services.yml` | Start all Airavata services | +| `stop_services.yml` | Stop all Airavata services | -## Supported OS with versions. +## Common Operations -- Centos 7 -- Rocky Linux 8 -- The PGA should also work on Ubuntu 16 +### Updating Existing Deployments -## Roles +#### Development Server +```bash +cd dev-tools/ansible +source venv/bin/activate -- **env_setup** :- Create user and group, install oracle java 8, open firewall - ports. -- **zookeeper** :- Download and install zookeeper. -- **rabbitmq** :- Download and install rabbitmq as service. -- **database** :- Download and install mysql(mariadb) as a service. -- **common** :- Checkout Airavata source from git and run maven build. Move - keystore files. -- **gfac** :- Setup and deploy Gfac component. -- **registry** Setup and deploy registry component. -- **api-orch** :- Setup and deploy Api-Orch components. -- **pga** :- Setup and deploy Airavata PHP Gateway. -- **keycloak** :- Setup and deploy Keycloak Identity management server. (Note: - Check roles/keycloak/README.md for details) +# Update services (stops, rebuilds, redeploys, starts) +ansible-playbook -i inventories/dev airavata_update.yml --ask-vault-pass -## Useful commands +# Or use vault password file +ansible-playbook -i inventories/dev airavata_update.yml --vault-pass-file=./vault-password.txt +``` -- Deploy database: - `ansible-playbook -i inventories/path/to/inventory/dir database.yml` -- Deploy Airavata middleware: - `ansible-playbook -i inventories/path/to/inventory/dir airavata.yml` -- Deploy Keycloak IAM server: - `ansible-playbook -i inventories/path/to/inventory/dir keycloak.yml` -- Deploy PGA: `ansible-playbook -i inventories/path/to/inventory/dir pga.yml` -- Deploy everything: - `ansible-playbook -i inventories/path/to/inventory/dir site.yml` +#### Production Server +```bash +cd dev-tools/ansible +source venv/bin/activate -## Configurations +# Update services (stops, rebuilds, redeploys, starts) +ansible-playbook -i inventories/prod airavata_update.yml --ask-vault-pass -- copy the `inventories/template` directory and modify CHANGEME values +# Or use vault password file +ansible-playbook -i inventories/prod airavata_update.yml --vault-pass-file=./vault-password.txt +``` + +### Starting/Stopping Services + +```bash +# Start all services +ansible-playbook -i inventories/dev start_services.yml --ask-vault-pass + +# Stop all services +ansible-playbook -i inventories/dev stop_services.yml --ask-vault-pass +``` + +### Initial Setup (First Time) + +For setting up a new deployment from scratch, see [DEVELOPER_GUIDE.md](DEVELOPER_GUIDE.md#setting-up-a-new-environment). + +## Setup Flow Overview + +The `airavata_setup.yml` playbook executes roles in the following order: + +``` +1. env_setup → System user, firewall, basic requirements +2. java → Java installation +3. common → Maven, Git, Airavata source checkout +4. zookeeper → Zookeeper installation +5. kafka → Kafka installation +6. rabbitmq → RabbitMQ installation +7. database → MariaDB installation and database setup +8. letsencrypt → SSL certificate generation +9. keycloak → Keycloak IAM server (24.0.0+ uses Quarkus) +10. reverse_proxy → Apache2 reverse proxy for Keycloak +11. api-orch → HAProxy for API server SSL termination +12. airavata_services → Build and deploy all Airavata services +``` + +**For detailed information about each role, dependencies, and multi-host configuration, see [SETUP_FLOW.md](SETUP_FLOW.md).** + +## Setting Up a New Deployment + +If you need to spin up a new deployment (new environment, new server, etc.), follow the [DEVELOPER_GUIDE.md](DEVELOPER_GUIDE.md#setting-up-a-new-environment). + +### Quick Summary + +1. **Copy template inventory:** + ```bash + cp -r inventories/template inventories/my-env + ``` + +2. **Edit configuration files:** + - `inventories/my-env/hosts` - Server IPs and host groups + - `inventories/my-env/group_vars/all/vars.yml` - Non-sensitive variables (ports, paths, versions) + - `inventories/my-env/group_vars/all/vault.yml` - **Sensitive variables** (passwords, API keys, database URLs) + - `inventories/my-env/host_vars//vault.yml` - Host-specific sensitive variables (SSH credentials) + + +3. **Key properties to change in `vault.yml`:** + - All `CHANGEME_*` values (database passwords, IAM passwords, OAuth secrets, etc.) + - Database hostnames/IPs in JDBC URLs + - Keycloak server URL (`iam_server_url`) + - Keycloak admin password (`keycloak_master_account_password`) + - Keycloak database password (`keycloak_db_password`) + - Keycloak client secrets (`keycloak_pga_client_secret`, `keycloak_jupyterlab_client_secret`, `keycloak_cilogon_client_secret`) + - Keycloak redirect URIs (update hostnames for your environment) + - Email credentials + - Keystore passwords + + +4. **Encrypt vault files:** + ```bash + ansible-vault encrypt inventories/my-env/group_vars/all/vault.yml + ansible-vault encrypt inventories/my-env/host_vars//vault.yml + ``` + +5. **Run setup:** + ```bash + ansible-playbook -i inventories/my-env airavata_setup.yml --ask-vault-pass + ``` + +## Supported Operating Systems + +- **Ubuntu**: 20.04, 22.04, 24.04 +- **CentOS**: 7 +- **Rocky Linux**: 8 + +## Documentation + +- **[DEVELOPER_GUIDE.md](DEVELOPER_GUIDE.md)** - Complete guide for developers: + - Setting up new environments + - Vault file management (encrypt, decrypt, edit) + - Service management (start, stop, update) + - Configuration file locations + - Troubleshooting common issues + + +- **[SETUP_FLOW.md](SETUP_FLOW.md)** - Detailed technical documentation: + - Role execution order and dependencies + - Role-by-role breakdown + - Multi-host deployment configuration + - Network requirements + - Variable reference + +## Key Roles + +- **env_setup** - System user, firewall, basic requirements +- **java** - Java installation (OpenJDK) +- **common** - Maven, Git, Airavata source checkout +- **zookeeper** - Zookeeper installation +- **kafka** - Kafka installation +- **rabbitmq** - RabbitMQ installation (uses distro packages) +- **database** - MariaDB installation and database setup +- **letsencrypt** - SSL certificate generation (Let's Encrypt) +- **keycloak** - Keycloak IAM server (24.0.0+ uses Quarkus with MariaDB driver) +- **reverse_proxy** - Apache2 reverse proxy for Keycloak +- **api-orch** - HAProxy for API server SSL termination +- **airavata_services** - Build and deploy all Airavata services + +## Troubleshooting + +See [DEVELOPER_GUIDE.md](DEVELOPER_GUIDE.md#troubleshooting) for common issues and solutions. + +## Legacy Playbooks + +The following playbooks are still available but deprecated in favor of `airavata_setup.yml`: +- `database.yml` - Database setup only +- `airavata.yml` - Airavata services only +- `keycloak.yml` - Keycloak setup only +- `site.yml` - Master playbook (includes all above) diff --git a/dev-tools/ansible/SETUP_FLOW.md b/dev-tools/ansible/SETUP_FLOW.md new file mode 100644 index 0000000000..e8cde5dca9 --- /dev/null +++ b/dev-tools/ansible/SETUP_FLOW.md @@ -0,0 +1,449 @@ +# Airavata Setup Flow and Multi-Host Configuration + +This document explains the setup flow and how to configure Airavata components across multiple hosts. + +## `airavata_setup.yml` Execution Flow + +### Execution Order + +``` +1. env_setup + ↓ +2. java + ↓ +3. common + ↓ +4. zookeeper + ↓ +5. kafka + ↓ +6. rabbitmq + ↓ +7. database + ↓ +8. letsencrypt + ↓ +9. keycloak + ↓ +10. reverse_proxy + ↓ +11. api-orch (conditional) + ↓ +12. airavata_services +``` + +### Role Details + +#### 1. `env_setup` +**Purpose**: System-level environment setup +- Creates system user/group (`user`, `group` variables) +- Configures firewall (firewalld/ufw) +- Sets up basic system requirements +- **Runs on**: `airavata_servers` (or respective host groups) + +#### 2. `java` +**Purpose**: Java installation +- Installs OpenJDK or Oracle JDK +- Sets `JAVA_HOME` +- **Runs on**: Any host that needs Java (Airavata, Keycloak) + +#### 3. `common` +**Purpose**: Common development tools +- Installs Maven +- Installs Git +- Checks out Airavata source code +- **Runs on**: `airavata_servers` only (needed for building Airavata) +- **Become user**: `{{ user }}` (non-root) + +#### 4. `zookeeper` +**Purpose**: Zookeeper installation +- Downloads and installs Zookeeper +- Configures `zoo.cfg` +- Creates systemd service +- **Runs on**: `[zookeeper]` group (usually same as `airavata_servers`) + +#### 5. `kafka` +**Purpose**: Kafka installation +- Downloads and installs Kafka +- Configures `server.properties` +- Creates systemd service +- **Runs on**: `[kafka]` group (usually same as `airavata_servers`) + +#### 6. `rabbitmq` +**Purpose**: RabbitMQ installation +- Installs RabbitMQ from distribution packages (Ubuntu/Debian) or RPM repositories (CentOS/Rocky) +- For Ubuntu 24.04: Uses distro packages (erlang + rabbitmq-server) +- Creates vhosts and users +- Configures permissions +- **Runs on**: `[rabbitmq]` group (usually same as `airavata_servers`) + +#### 7. `database` +**Purpose**: MariaDB installation and database setup +- Installs MariaDB +- Sets root password +- Creates databases: + - `experiment_catalog` + - `app_catalog` + - `replica_catalog` + - `workflow_catalog` + - `sharing_catalog` + - `credential_store` + - `profile_service` + - `research_catalog` + - `keycloak` (if Keycloak role runs) +- Creates database user (`db_user` with `db_password`) +- **Runs on**: `[database]` group (can be separate host) + +#### 8. `letsencrypt` +**Purpose**: SSL certificate generation +- Installs certbot +- Generates Let's Encrypt certificates for: + - Keycloak vhost (`keycloak_vhost_servername`) + - API server (`api_server_public_hostname`) +- **Runs on**: Hosts that need SSL certificates + +#### 9. `keycloak` +**Purpose**: Keycloak IAM installation +- Downloads Keycloak distribution (24.0.0+ uses Quarkus) +- Installs MariaDB JDBC driver in `providers/` directory (for Keycloak 24+) +- Configures database connection via environment variables (can be remote) +- Sets up SSL certificates +- Creates admin user via environment variables +- Imports realm definition (default realm) if `keycloak_realm_import_enabled` is true +- Creates systemd service +- **Runs on**: `airavata_servers` (by default) or `keycloak_servers` (if separate) +- **Note**: Requires database, Java, and SSL certificates +- **Keycloak 24+**: Uses Quarkus, requires MariaDB driver in providers directory, uses environment variables for configuration + +#### 10. `reverse_proxy` +**Purpose**: Apache2 reverse proxy for Keycloak +- Installs Apache2/httpd +- Configures virtual host to proxy to Keycloak +- Enables SSL +- **Runs on**: Same host as Keycloak + +#### 11. `api-orch` +**Purpose**: HAProxy for API server SSL termination +- Installs HAProxy +- Configures SSL termination +- Proxies to API server on port 8930 +- **Runs on**: `airavata_servers` +- **Condition**: Only runs if `api_server_public_hostname` is defined + +#### 12. `airavata_services` +**Purpose**: Build and deploy Airavata services +- Builds Airavata from source (Maven) +- Generates configuration files from templates +- Deploys services to `deployment_dir` +- Starts all services (API Server, Orchestrator, Registry, Agent Service, Research Service, File Server, REST Proxy) +- **Runs on**: `airavata_servers` only +- **Become user**: `{{ user }}` (non-root) + + +## Multi-Host Configuration + +### Supported Separations + +You can separate: +- **Keycloak** → Separate host +- **Database** → Separate host +- **Airavata Services** → One host (all services together) + +### Example: Three-Host Setup + +**Host 1**: Database Server (MariaDB only) +**Host 2**: Keycloak Server (Keycloak + Apache2) +**Host 3**: Airavata Server (All Airavata services + Zookeeper + Kafka + RabbitMQ) + +## Configuration Steps + +### Step 1: Update Inventory (`inventories/my-env/hosts`) + +```ini +# Database server (separate host) +[database] +db-server ansible_host= ansible_user=exouser + +# Keycloak server (separate host) +[keycloak_servers] +keycloak-server ansible_host= ansible_user=exouser + +# Airavata services server +[airavata_servers] +airavata-server ansible_host= ansible_user=exouser + +# Infrastructure services (on Airavata server) +[zookeeper] +airavata-server + +[kafka] +airavata-server + +[rabbitmq] +airavata-server +``` + +### Step 2: Database Server Configuration + +Create `inventories/my-env/group_vars/database/vars.yml`: + +```yaml +--- +# Database server doesn't need much, but you can set: +# (Most variables are in group_vars/all/vault.yml) +``` + +Create `inventories/my-env/group_vars/database/vault.yml`: + +```yaml +--- +# MariaDB root password +mysql_root_password: "CHANGEME_MYSQL_ROOT_PASSWORD" + +# Database user for Airavata services +db_user: "airavata" +db_password: "CHANGEME_DB_PASSWORD" +``` + +### Step 3: Keycloak Server Configuration + +Create `inventories/my-env/group_vars/keycloak_servers/vars.yml`: + +```yaml +--- +# Keycloak host-specific variables +keycloak_vhost_servername: "auth.airavata.apache.org" # DNS name for Keycloak +letsencrypt_email: "admin@airavata.org" + +# Keycloak database connection (points to database server) +keycloak_db_host: "" # Database server IP +keycloak_db_port: "3306" +keycloak_db_schema_name: "keycloak" +``` + +Create `inventories/my-env/group_vars/keycloak_servers/vault.yml`: + +```yaml +--- +# Keycloak admin credentials +keycloak_master_account_username: "admin" +keycloak_master_account_password: "CHANGEME_KEYCLOAK_ADMIN_PASSWORD" + +# Keycloak database credentials +keycloak_db_username: "keycloak" +keycloak_db_password: "CHANGEME_KEYCLOAK_DB_PASSWORD" +``` + +### Step 4: Airavata Server Configuration + +Update `inventories/my-env/group_vars/all/vars.yml` (if needed): + +```yaml +--- +# Airavata version and build settings +airavata_version: "0.21-SNAPSHOT" +git_branch: "master" +deploy_user: "exouser" +deployment_dir: "/home/{{ deploy_user }}/airavata-deployment" +``` + +Update `inventories/my-env/group_vars/all/vault.yml`: + +```yaml +--- +# Database URLs +registry_jdbc_url: "jdbc:mariadb://:3306/experiment_catalog" +appcatalog_jdbc_url: "jdbc:mariadb://:3306/app_catalog" +replicacatalog_jdbc_url: "jdbc:mariadb://:3306/replica_catalog" +workflowcatalog_jdbc_url: "jdbc:mariadb://:3306/workflow_catalog" +sharingcatalog_jdbc_url: "jdbc:mariadb://:3306/sharing_catalog" +profile_service_jdbc_url: "jdbc:mariadb://:3306/profile_service" +credential_store_jdbc_url: "jdbc:mariadb://:3306/credential_store" + +# Agent and Research service datasources +agent_service_datasource_url: "jdbc:mariadb://:3306/app_catalog" +research_service_datasource_url: "jdbc:mariadb://:3306/research_catalog" + +# IAM credentials (Keycloak) - points to Keycloak server +iam_server_url: "https://auth.airavata.apache.org" # Keycloak hostname +iam_admin_username: "admin" +iam_admin_password: "CHANGEME_IAM_PASSWORD" + +# Database passwords (for connecting to database server) +registry_jdbc_password: "CHANGEME_DB_PASSWORD" +appcatalog_jdbc_password: "CHANGEME_DB_PASSWORD" +# ... (all use same db_password) +agent_service_datasource_password: "CHANGEME_DB_PASSWORD" +research_service_datasource_password: "CHANGEME_DB_PASSWORD" +``` + +### Step 5: Running Setup + +**Option A: Run setup separately for each host** + +```bash +# 1. Setup database server first +ansible-playbook -i inventories/my-env airavata_setup.yml \ + --limit database \ + --tags "env_setup,database" \ + --ask-vault-pass + +# 2. Setup Keycloak server +ansible-playbook -i inventories/my-env airavata_setup.yml \ + --limit keycloak_servers \ + --tags "env_setup,java,letsencrypt,keycloak,reverse_proxy" \ + --ask-vault-pass + +# 3. Setup Airavata server (skip database and keycloak) +ansible-playbook -i inventories/my-env airavata_setup.yml \ + --limit airavata_servers \ + --skip-tags "database,keycloak,iam" \ + --ask-vault-pass +``` + +**Option B: Use `--limit` to target specific hosts** + +The playbook will automatically skip roles that don't apply to the target host based on inventory groups. + +## Network Requirements + +### Database Server +- **Port 3306**: Must be accessible from: + - Keycloak server + - Airavata server +- **Firewall**: Allow MySQL connections from Keycloak and Airavata IPs + +### Keycloak Server +- **Port 443 (HTTPS)**: Must be accessible from internet +- **Port 80 (HTTP)**: For Let's Encrypt verification +- **Outbound**: Must connect to database server (port 3306) +- **DNS**: `keycloak_vhost_servername` must resolve to Keycloak server IP + +### Airavata Server +- **Port 8930**: API Server (or via HAProxy on 443) +- **Port 8940**: Orchestrator +- **Port 8962**: Profile Service +- **Port 8970**: Registry +- **Port 8960**: Credential Store +- **Port 7878**: Sharing Registry +- **Port 18880**: Agent Service +- **Port 18899**: Research Service +- **Port 8050**: File Server +- **Port 8082**: REST Proxy +- **Port 2181**: Zookeeper +- **Port 9092**: Kafka +- **Port 5672**: RabbitMQ +- **Outbound**: Must connect to: + - Database server (port 3306) + - Keycloak server (HTTPS) + +## Running Setup + +### Full Setup (All on Same Host) +```bash +ansible-playbook -i inventories/dev airavata_setup.yml --ask-vault-pass +``` + +### Updating Existing Deployment +```bash +# Development server +ansible-playbook -i inventories/dev airavata_update.yml --ask-vault-pass + +# Production server +ansible-playbook -i inventories/prod airavata_update.yml --ask-vault-pass +``` + +### Multi-Host Setup (Separate Hosts) +```bash +# Setup in order: Database → Keycloak → Airavata +ansible-playbook -i inventories/dev airavata_setup.yml \ + --limit database \ + --tags "env_setup,database" \ + --ask-vault-pass + +ansible-playbook -i inventories/dev airavata_setup.yml \ + --limit keycloak_servers \ + --tags "env_setup,java,letsencrypt,keycloak,reverse_proxy" \ + --ask-vault-pass + +ansible-playbook -i inventories/dev airavata_setup.yml \ + --limit airavata_servers \ + --skip-tags "database,keycloak,iam" \ + --ask-vault-pass +``` + +## Troubleshooting + +### Common Issues + +1. **Database connection fails** + - Check `mysql_root_password` is set correctly + - Verify MariaDB is running: `systemctl status mariadb` + - Check firewall allows connections from Keycloak and Airavata hosts + - Verify database server IP is correct in JDBC URLs + +2. **Keycloak fails to start** + - Verify database is created: `mysql -u root -p -e "SHOW DATABASES;"` + - Check Keycloak can connect to database: `telnet 3306` + - Check Keycloak logs: `journalctl -u keycloak -f` + - Verify SSL certificates exist + - Verify `keycloak_db_host` points to correct database server + - For Keycloak 24+: Verify MariaDB driver exists: `ls -la /home/airavata/keycloak-24.0.0/providers/mariadb-java-client.jar` + - Check environment variables: `systemctl show keycloak | grep KC_DB` + +3. **Airavata services fail to start** + - Check database connections in `vault.yml` (should point to database server IP) + - Verify RabbitMQ vhost exists: `rabbitmqctl list_vhosts` + - Check service logs in `deployment_dir/*/logs/` + - Verify `iam_server_url` points to Keycloak server + - Use `start_services.yml` to start services: `ansible-playbook -i inventories/dev start_services.yml --ask-vault-pass` + - Use `stop_services.yml` to stop services: `ansible-playbook -i inventories/dev stop_services.yml --ask-vault-pass` + +4. **SSL certificate generation fails** + - Verify DNS points to server: `dig keycloak_vhost_servername` + - Check port 80/443 are open: `ss -tuln | grep -E '80|443'` + - Verify email is valid: `letsencrypt_email` + +5. **Network connectivity issues** + - Test database connection: `mysql -h -u root -p` + - Test Keycloak connection: `curl https://auth.dev.cybershuttle.org` + - Check firewall rules: `firewall-cmd --list-all` or `ufw status` + +## Next Steps After Setup + +1. **Verify services are running**: + ```bash + # Database server + ansible database -i inventories/dev -m shell \ + -a "systemctl status mariadb" + + # Keycloak server + ansible keycloak_servers -i inventories/dev -m shell \ + -a "systemctl status keycloak apache2" + + # Airavata server + ansible airavata_servers -i inventories/dev -m shell \ + -a "systemctl status zookeeper kafka rabbitmq-server" + ``` + +2. **Check Airavata services**: + ```bash + ansible airavata_servers -i inventories/dev -m shell \ + -a "ps aux | grep java | grep airavata" + ``` + +3. **Update services** (use `airavata_update.yml`): + ```bash + ansible-playbook -i inventories/dev airavata_update.yml \ + --limit airavata_servers \ + --ask-vault-pass + ``` + +4. **Start/Stop services independently**: + ```bash + # Start all Airavata services + ansible-playbook -i inventories/dev start_services.yml --ask-vault-pass + + # Stop all Airavata services + ansible-playbook -i inventories/dev stop_services.yml --ask-vault-pass + ``` \ No newline at end of file diff --git a/dev-tools/ansible/airavata_setup.yml b/dev-tools/ansible/airavata_setup.yml new file mode 100644 index 0000000000..0429e273f8 --- /dev/null +++ b/dev-tools/ansible/airavata_setup.yml @@ -0,0 +1,118 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +# Airavata Full Environment Setup Playbook +# +# This playbook sets up a complete Airavata environment from scratch including: +# - Environment setup (users, groups, firewall) +# - Java and Maven installation +# - Zookeeper installation and configuration +# - Kafka installation and configuration +# - RabbitMQ installation and configuration +# - MariaDB installation and configuration +# - SSL certificate setup (Let's Encrypt) +# - Airavata services build and deployment +# +# Usage: +# ansible-playbook -i inventories/ airavata_setup.yml --ask-vault-pass +# +# Prerequisites: +# - Clean server with root/sudo access +# - DNS configured (for Let's Encrypt) + +- name: Full Airavata Environment Setup + hosts: airavata_servers + become: yes + + roles: + # Environment and prerequisites + - role: env_setup + tags: + - env_setup + + # Java installation + - role: java + tags: + - java + + # Maven, Git, and source checkout + - role: common + become: yes + become_user: "{{ user | default('airavata') }}" + tags: + - common + + # Zookeeper installation and configuration + - role: zookeeper + tags: + - zookeeper + - airavata + + # Kafka installation and configuration + - role: kafka + tags: + - kafka + - airavata + + # RabbitMQ installation and configuration + - role: rabbitmq + tags: + - rabbitmq + - airavata + + # MariaDB installation and configuration + - role: database + tags: + - database + + # SSL certificates (Let's Encrypt) + - role: letsencrypt + tags: + - ssl + - letsencrypt + + # Keycloak IAM installation and configuration + - role: keycloak + tags: + - keycloak + - iam + + # Reverse proxy (Apache2) + - role: reverse_proxy + tags: + - reverse_proxy + - apache + + # HAProxy for API server reverse proxy (SSL termination) + - role: api-orch + tags: + - haproxy + - api_proxy + when: api_server_public_hostname is defined + + # Build and deploy Airavata services + - role: airavata_services + become: yes + become_user: "{{ user | default('airavata') }}" + tags: + - airavata_services + - deploy + diff --git a/dev-tools/ansible/airavata_update.yml b/dev-tools/ansible/airavata_update.yml new file mode 100644 index 0000000000..69d5f3eb6a --- /dev/null +++ b/dev-tools/ansible/airavata_update.yml @@ -0,0 +1,70 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +# Airavata Services Update Playbook +# +# This playbook updates existing Airavata services in environments where +# infrastructure (Zookeeper, RabbitMQ, databases) already exists. +# +# Usage: +# ansible-playbook -i inventories/ airavata_update.yml --ask-vault-pass +# +# Prerequisites: +# - Infrastructure components already running (Zookeeper, RabbitMQ, databases) +# - Previous deployment exists +# - SSH access to target servers + +- name: Update Airavata Services + hosts: airavata_servers + become: yes + become_user: "{{ deploy_user | default(ansible_user) }}" + + tasks: + - name: Display deployment information + debug: + msg: "Updating Airavata services on {{ inventory_hostname }}" + + - name: Stop all services + include_role: + name: airavata_services + tasks_from: stop_services + + - name: Pull and build Airavata + include_role: + name: airavata_services + tasks_from: build + + - name: Deploy all services + include_role: + name: airavata_services + tasks_from: main + vars: + skip_stop_services: true + + - name: Start all services + include_role: + name: airavata_services + tasks_from: start_services + + - name: Display completion message + debug: + msg: "Airavata services update completed on {{ inventory_hostname }}" + diff --git a/dev-tools/ansible/database.yml b/dev-tools/ansible/database.yml index 6c64d44102..a403a5bcb7 100644 --- a/dev-tools/ansible/database.yml +++ b/dev-tools/ansible/database.yml @@ -19,9 +19,9 @@ # --- -# Gather facts on the following +# Gather facts on the following (needed for database role to grant MySQL access from remote IPs) - hosts: api-orch -- hosts: helix +- hosts: keycloak - hosts: database tags: mysql , airavata diff --git a/dev-tools/ansible/inventories/dev/group_vars/all/vars.yml b/dev-tools/ansible/inventories/dev/group_vars/all/vars.yml new file mode 100644 index 0000000000..5320ff8697 --- /dev/null +++ b/dev-tools/ansible/inventories/dev/group_vars/all/vars.yml @@ -0,0 +1,134 @@ +--- +# Non-sensitive configuration variables for dev environment + +# Airavata version and build settings +airavata_version: "0.21-SNAPSHOT" +git_branch: "master" +airavata_git_repo: "https://github.com/apache/airavata.git" +airavata_source_dir: "/home/{{ deploy_user }}/airavata-src" +deployment_dir: "/home/{{ deploy_user }}/airavata-deployment" + +# Maven version (should match common role's apache_maven_version) +apache_maven_version: "apache-maven-3.9.11" + +# Deployment user +deploy_user: "airavata" + +# System user/group (used by env_setup and other roles) +user: "{{ deploy_user }}" +group: "{{ deploy_user }}" +user_home: "/home/{{ user }}" + +# Service ports +api_server_port: 8930 +api_server_tls_port: 9930 +profile_service_port: 8962 +registry_port: 8970 +registry_server_port: 8970 +sharing_registry_port: 7878 +cred_store_port: 8960 +agent_service_port: 18880 +research_service_port: 18899 +file_server_port: 8050 +restproxy_port: 8082 + +# Database names +app_catalog: "app_catalog" +exp_catalog: "experiment_catalog" +replica_catalog: "replica_catalog" +workflow_catalog: "workflow_catalog" +sharing_catalog: "sharing_catalog" +credential_store: "credential_store" +profile_service: "profile_service" +research_catalog: "research_catalog" + +# Database drivers +registry_jdbc_driver: "org.mariadb.jdbc.Driver" +appcatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +replicacatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +workflowcatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +sharingcatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +profile_service_jdbc_driver: "org.mariadb.jdbc.Driver" +credential_store_jdbc_driver: "org.mariadb.jdbc.Driver" + +# Database users +registry_jdbc_user: "{{ db_user | default('airavata') }}" +appcatalog_jdbc_user: "{{ db_user | default('airavata') }}" +replicacatalog_jdbc_user: "{{ db_user | default('airavata') }}" +workflowcatalog_jdbc_user: "{{ db_user | default('airavata') }}" +sharingcatalog_jdbc_user: "{{ db_user | default('airavata') }}" +profile_service_jdbc_user: "{{ db_user | default('airavata') }}" +credential_store_jdbc_user: "{{ db_user | default('airavata') }}" + +# Paths +local_data_location: "/home/{{ deploy_user }}/temp-storage" +file_server_storage_location: "/home/{{ deploy_user }}/temp-storage" +agent_service_storage_path: "/var/www/portals/gateway-user-data" + +# Keystore file name +vault_keystore_file: "airavata.sym.p12" + +# Email monitor settings +email_based_monitor_host: "imap.gmail.com" +email_based_monitor_store_protocol: "imaps" +email_based_monitor_folder_name: "INBOX" +email_expiration_minutes: 60 +email_based_monitoring_period: 10000 + +# Kafka and RabbitMQ +kafka_broker_url: "localhost:9092" +restproxy_broker_url: "localhost:9092" +rabbitmq_port: 5672 +rabbitmq_status_exchange_name: "status_exchange" +rabbitmq_process_exchange_name: "process_exchange" +rabbitmq_experiment_exchange_name: "experiment_exchange" +experiment_launch_queue: "experiment_launch" + +# Zookeeper connection +zookeeper_client_port: 2181 +zookeeper_connection: "localhost:{{ zookeeper_client_port }}" +embedded_zk: false +# Zookeeper AdminServer port (default is 8081 to avoid conflict with Keycloak on 8080) +# Can be overridden here if needed +# zookeeper_admin_server_port: 8081 + +# API Server public hostname (for HAProxy SSL termination) +api_server_public_hostname: "api.dev.cybershuttle.org" + +# API Server connection settings for Django +api_server_host: "{{ api_server_public_hostname }}" +api_secured: true + +api_server_bind_host: "0.0.0.0" +orchestrator_bind_host: "0.0.0.0" +registry_bind_host: "0.0.0.0" +sharing_registry_bind_host: "0.0.0.0" +cred_store_server_bind_host: "0.0.0.0" +profile_service_bind_host: "0.0.0.0" + +# TLS Configuration +tls_enable: false + +# Let's Encrypt email for certificate notifications +letsencrypt_email: "admin@cybershuttle.org" + +# Keycloak virtual host hostname (for Let's Encrypt certificate) +keycloak_vhost_servername: "auth.dev.cybershuttle.org" + +# Firewall subnets (for allowing access to services) +# These can be restricted to specific IP ranges for security +# For test environment, using 0.0.0.0/0 to allow all (adjust for production) +zk_subnets: + - "0.0.0.0/0" +kafka_subnets: + - "0.0.0.0/0" +rabbitmq_subnets: + - "0.0.0.0/0" +db_subnets: + - "0.0.0.0/0" + +# Other non-sensitive configuration +enable_sharing: true +enable_validation: true +enable_realtime_monitor: true +job_notification_enable: true diff --git a/dev-tools/ansible/inventories/dev/group_vars/all/vault.yml b/dev-tools/ansible/inventories/dev/group_vars/all/vault.yml new file mode 100644 index 0000000000..4a9db2a276 --- /dev/null +++ b/dev-tools/ansible/inventories/dev/group_vars/all/vault.yml @@ -0,0 +1,531 @@ +$ANSIBLE_VAULT;1.1;AES256 +32343562636565363430396138346263326432616635623835376133663965343364653464633134 +3737646334383034353333633630643239663838333335620a633638323765333332323739373261 +61363932326237663030646230386631653864633361646234353133323830363366656333393732 +6564383031386536300a303133346234653338316565643463376262393630363265393038353238 +38643138363237363264336137613663306263383236346562363033643532303435633966653864 +37616432623634643463613563383732393939663165643934353762616632323530363933636636 +30613839623339666237306137653036373531373530303662363332323235626634383032333166 +31383435333063613865386462653438623036643637376534373535363137643762393336306532 +32346164383661383934383639353039343631663234363664356338396135353166616665623261 +65613536616230353730396239646666656534613864666337363935313233363237653536336136 +61666333323538323364373434363736616634363666633166346235333030626665396337313638 +34656136666664333236383535303639613363343733353566666334316461656533666461353139 +64613136326239393736346664613765653532396165386264333530393033653633393537666134 +61323962663735376662303936376333616539363962353538363733383937393532383337363630 +34343139333661663165316338366434623562666166313030323361363832306464363135316234 +36353734356631653537333264623735303964356230643739633939373930313032623061643132 +32393766313961363533383331333336396531353639613863376239663931336462363663613162 +62376462303732646535383462393331353638666465663764386632303830383464636339303830 +35643062373233616234343734383038376431643563363464383737356561666663366361303033 +61666537326563373335326261636539663234376238356432373735366633373538343139363062 +66633738663136353034313932333433333964303338646433333361636537343032346362303762 +37363431616131353030333338663033386137653133393532666434613833633039333063646633 +33316563303636666165653862613239333932373138303663316235323037623563363166633261 +35383231396265396430613730366265346266666138636330333538616361336161626561386633 +37356230633830376538376233376237333462316362383265316235306639636536646261633436 +35643466306361386637363532313565313062396466366439396263326662363239623635616335 +38663333363762396334323865366361313031343932633830346165353831636131313832393735 +65356364636162336436653063393833366263376139373261633636356139303336613531613038 +31636561653334366230663730323061373837643635376462616631306537663133633266316632 +66336335646234373136613839373765633535373063616535636635326430366639313138636131 +33613762306639373831613565666430383737366433363464646339633231316130373066633436 +36323266393333353837323131356236353232626638346434343135623933653661643034393531 +37353231333537326634303433663466306461363765653036643938326663303134323432633336 +64313337623262336637613234366634323062626438383864653866343137643862633763663661 +64356637333638353639646166336166373163653761353131623435343039346563653462656464 +35356630373033663962326365666231653464636265646332623231396461346138323665323337 +61326633386433343761356338363764373034376465373065623862626533616238393730376235 +64636264343832386561646132383133393538326364336635613365363033353766623435306231 +66646438613931356261653635363635653931353964633439346637333331653135323838613362 +61663932333263356265623266396439333532626364633964356365373961303632646631306662 +61653537623537633461636164383431333866333935626530303833643262656362616264656431 +63386335363633653530316335336534313232396664346639376566626134316338623431356336 +30393930373437616336623661663532386631323965363863363030313937356431626337656639 +38366132353465326162373232643363316133646265663830353463613963303238353066303834 +34303762636430313636373833366663373038363032343162616163363837636361326631623439 +63366534636465653137613738363833363762323931363636303365613938663561633063396262 +63633038643765313363336430333631366239623437316333396439313564653835363233303566 +63636333366635623636343831326233633438633633613532623336646332363832363362363830 +38383262363166353338303839336537343235303163373337653535643363386564303239303537 +35326165336463386561313530326138363937316466376363343831386434643062316663336663 +34643738653034613133363232376566636636656233383264383364393533626137613865383232 +31363366623361613365346235323961616239353638363563623238393233643334383837323531 +61326633656562383038393830383632306138343662373331353964363862613436636538356230 +36393134313834663961663139663966613663663639353333663538393262336637653330316266 +63313838626639313835393939336333626337613534393638666365303738383935633233653262 +63313432396561366538333631646337316263393538326136616538653261363937633466346131 +63303962393134356630343131616132633939666239386466326438633738333134633462383162 +65313832663362663064313637616134376465363631663438303132613361356134353461323230 +61383435663935396133333263323333396335393761373935326163636239653064333833636461 +63386638386562373234656632616330373335323231333634623161363266393538383633376530 +61636532323338323463613261336334303830333835323636616537386532663434376137326135 +35326263363934613237343261336135373665633033303631326139333430326161633833396134 +32393036623862396532316466396436386539636366353333376336313836326434383934373134 +66653236653532653662653836616664343230323262326334323632323664303065373733383038 +36613434393936323963363236303232623733396464343236373833323966633839346639646333 +38303162636364373563373462393839626538343966313539373466366563333536376463353738 +37303365373634316635386639613236373831653933366630353563323634643836363161666236 +38323930306637323237643533373932366436376133373337666165663031663434363135376264 +61393661643764356563326536396130356636333335333038366134353263373034643336626637 +61306665643231373161306466646237393939636331333636363233323531353062336433353433 +36353165313163626165666131653034343465383430623061333862653462376530313663656165 +31383232393033376434346438373263363366363335653334373165306434346564653666396630 +36316136393238386565663736636466323531363333386565303263653138316662333564393635 +30346264346538623361323734326662663363356337643830626564373865353430653131343534 +38336239376461636633353264326632353138666639353465373261383964633432363639636639 +37656632626433626337353331613062303030386166633339393166313330363435616631356639 +34303134653939346266393831313530643564306364653463353066326461353864326462373462 +33363761366137306263646366373531326339616462656339353633396366343837383432653537 +63383833393064373336333631353237653363323732653162623731623162616133303762323564 +36663435646335393565396237393038643563663538633136343866343362313239666232333339 +33373262343565303930646331636333653031323338396232633136636661363466663830326233 +35626132323063643030393261636236646265336438626537376135643766393963326139316339 +30626166376362646230363364383732333465346564303766616264336338336566653664353263 +62306461636361323861313139623932323733373663343033383134323530343537626661376662 +62383634623934313339643361643565363733623066663031373862383134656339313661326264 +37336263366638303066386464346530316131373965383033643463613034633433653964313530 +31356364373135303763393438626139373134336237306539316139613938613762656134616133 +32373962636630623437306430616361306333623433366563383264323037336566656233356435 +64616461393335316465333462356432313538633064303539363561353030366364313839303137 +38653838316636613063393330373466643937343933303931386531613266343433386534643833 +61616466303161626666303465623430646565663461396138356230666431346539356231653734 +38626663386134393863643938316136383134303839646635383364626231633564613735373333 +61323435616639346532323136656164653165393064656638306666653431646231373639323738 +34376362623365343066623239316633306439356565366138356537303236306664636431366462 +37643531343332303462643937346438373431613865356236613463656466323065346435363634 +62376464303166643233373161623865636635316232313437636639363836663664353865333639 +63323835643531366533363539353332363766333637633838383530646561393061323337366464 +34663232383939336565613961653332613664356438623034303463323437373766383238373830 +38366365356362313764656462393034633337343933323264313835366638363464343838613733 +32393833356466383765303332326431333330326633646163386336376239323231633062626164 +65396538643333666639333137376638303634393131346438633039303537613563316533663133 +63363837636164376632626334666232666264363263316636366439306434336366356364643030 +66366631613063303764353333356663643439373235313963363466656639333163636331626166 +66373461346264633966353935613036333733306466376363633236356235636661336133313965 +62633031643764316465626338653461623239623130323165363366633734393938626630393461 +30353463623962323565636230313535643065646337373664666261386532373863663737663163 +33663264666434613930653234663062626536343731393965306237323435656238383035336163 +38616564393564343861336366313862616432616261613265313530663037343636646536656566 +35363333626531336335353564616637666331366538663464373134616632366130653264386365 +38613738313962333738336234636430666237643834393835363261653231373032653835333130 +36663330383235303239393532383833653736333635393731663232656261373838633164343537 +61666332636264343463356166383539613565353339323333326233383863663531303935313966 +62393132623334316237653062373834623036373261306137313232383135393332353664333632 +36393336313461646632623837386633313763656632366130656337306638623731623063633063 +61616238643434366332383530316431623635613833663033306133313962613239613935653765 +63633463623461643061663338623035326237373937306665636335333463343537333037386231 +32646561333039353062623332663833383937373764626539343030303037303939353266383366 +66313666636136626330646364316437316366313466306336616666396463653830663064633762 +65376562346466646265303239323330623839643332353334656237616234386137623131323439 +64306237646133613936326132353834353132393435353261386561613032383634643463663232 +66373336316362376139626663663565626530663138326539653664613036393366316239303736 +30626638333462373639363738326131303063393433323432373432316436353662613337373961 +36626439666639316532316533656433316430366636393634643139316265343931623234356132 +35303230666138386237613366613930313831633831363030633530663363366561373037636633 +36363934313264386238373231356338356633303264366238663234366465653738336663663364 +30343031336633653832373361626663663739643838306232383263313531343030396662396232 +39303161643561303962633962653232386164303237656135346634313638616265333635663865 +33366262323439303734363662346339333434386533613933666161623965366330343836323131 +39303832666334353031633430633739623237313934663266666264396464353633383461383530 +30613265643232326235343939643265393264316533656533376630633830623661353066316262 +36613262663865326631313365623635626537623064303932336131356435646539663738316637 +37313430353264373762346463663438616462636334356336313265643262313039633139396430 +39616434343065326164383664323262363864373261613266653563303834346536666366363465 +30383331613064653965303037373464386433376166656137623564626265643630643165366565 +34616165636365313163666331623134396130653837663637616366373731626637376362656261 +38343132616464343230646262353934393065656536323464663163616164646438613335343664 +63313635376264333834316634643365336365343535303033326237386237613562386339646238 +38396332323063343161363966663861636563356638383432376463643839326666656166316366 +36306630643965663836343230313766343765623032343566333962323339336233356636626332 +61643562333038393839663162386331376139303937386537383635633964343466303232313662 +38373330363966343563663434616432386635633965303864663133643131346137326162613339 +65393165306537363962666163386434363635333566316361636561383135366562303965376466 +61386366376238336333326264653838666137306435333435376364343437376530353939363862 +39353532333162353364633563353231303265653662626134376434656137353334333866643935 +30643938373663376537393262376331393232353137633831663063623266636537353065396234 +61333731306237336532313866623637383662616461303835373734303063396537633438303739 +36666631333932313732323736646333633366653431346139353463643261323533326431643934 +37323133353062353064336239346336333133663130363363633235366331343134643133303833 +34346438363562653332343736366164383433356365623063623836393131643463373830663565 +38303539333764343430386339343937366332626433303064643763653964623330623837623036 +63306632653732303230623561323136306635363833356235356330393362663261663430373939 +63636430303536663637323666306231393236353835633662393333363362393539346131313635 +62616130333336646161316561303537303138636333363137353662336133623235303131353463 +61313534663433333461333362646366643736343465323061626263303866653132663466393739 +65656638636366666335666461636363636134356331636437353837303434653332343938363264 +38656163343034626138376634633636316631316130636537633639313662666466353961393066 +30373731626165346666313233626437613561613139656332663538366563613164353236353933 +61393761383231313738373838666562643439353734653935666462333238373330353566346666 +35386263383064663336623437316639383437653939613064363932386430653666383564363763 +31363933386432336265343462656630623131646130626565356565353936326361353036333462 +63393233653762306331623564336162303230376333376262633034393136613765333538646335 +63343136326565356165636636643133313261663533323364656436653061393131636332653037 +39636633336138623362616338383166336339373835396164656434623031386132373130323931 +31633565653061356562616563386533623836363039643863343736373366653161346236333731 +66396237646664313336353231303237643266646265323735666165363937396138316530633863 +38333362393665353338636437396131666339633639363062346333643334376234633462303133 +65663866373361376562383263383035303334336332333865383233373339343766623531373532 +65663665366339663534623764306435636666383436666366363739303762613738396261353632 +61393966623263373531353032666337393232383764363365326661663238663135363434383034 +34373864663530633563383438323462666138366537656233306336303034333739326161383865 +63393736626235623032653238303432653936356338346333353163363764666137383034386636 +37643839356636616531613133666534346132626233366161656163303464356362363064353565 +61356232313638313432343762393532333065646164656635323639313963356563363934313463 +66336161656235633564356236383332376631333563633339306335346365333536376139663733 +36623261653264353330613536303333653862363463333135326166663335653965643966623033 +38363339656530653334366665656130633937343637326437656139666234343135663730623739 +64393364303834316232343530663436653565363666333635613465633862663232313132626462 +35333761623764656139376462333335343565336265343533363065323532373662313235346238 +30653930633266363237396636326536616638363731383863303564343538663961623635653136 +37323962336132653530363361643637656136343864373736356561336132306136616236626431 +37653561346262316533386463643063306438653466393034303062653163336364666137336162 +37353036636232366166313061373434646330373534346536643062366234356364633936646263 +38353736376234643966626538333930303630656632646630363237333831353536343061666362 +32366335306438306533346533363061383763656137396561376330346433343233643235363264 +61333632656439353262633232623038623364313238646231623834363435366638346130666239 +31313762366639336431376536383831323762323862353635353731633464633763333830613331 +37383636653939313365313164613664656339303739306363303535333961613033613865653064 +33323630643839363437646438356564646433373536386535303865623863663464346165636132 +37396266353531366266383837656364323134373935663763396636633565326363343132373462 +66376534363033633365626333303533376164376335666139353232376139646535396530386139 +37353362363332353635313331666664316639613535336434396238343138313463376232303763 +66326263356636323730376266303334663233383661346233353438336137643336666638616538 +38643831383631653937653735613939663265636136366163316139366238643735623161326266 +64613462663238373330323430383565376630306237363836393964326665393261633738306633 +36396639656463386434373964313464316138613439626436373930316536383038623062633135 +34303264373332663864393837373233643134376565393237396666313030386533346337373332 +33616466633737386361613037373363666239323530656131623964393237633361643966626561 +32373939356564646662316363336130643930383132643664373838633566303336333737653837 +65373137343931653630636662353736306465343765356165303433323931333363333930323933 +38343239646237666663353034656262386466666362326636353831373764356538363638343836 +62343830636162613534633539346436666534646465353732303863343135313166303135383564 +35633833316563336135656339373130366563353662616663333836343633383264613931656135 +63353430356364336530303838343332373938666165356162623238643666626238613663636138 +36336564353137613530386331396136623639306432626661633065373632666664653532383038 +39326336376139376635316439313933373231313131363637663162633363346633313232313562 +65393636306336633763373563366332613765346566336436333663313566353130353261343634 +35343534326637306531653537373761616134613238666536343239306439623730636461313566 +36386231613530313533666564313435353162616533663032393030336264346565646539616636 +32353439643838653936393333646634633163306235353066303235663263643732646232313635 +35323031636466313362663030396664623130396265313966656563383234666462393737316562 +39636532653162376531376537323132363663616665353139303537323638323462373061373036 +38633934353837303765663333306232633733643263616665626562343736666537396437656433 +37616635333833626363363836326535633261653336343635306432623335653364663262396265 +62653135343466326238356533383233663036343930653238663466663761373832663666613538 +31623030336261356262306362393332653836343439353537363530643961393565613964306132 +30333430313366316464323264393230396432623539626635386563633839306332376662393564 +37653462636164396639653162333432303563613738646465386362646366643531336535333735 +32363738613365333963383830346139656339393361663832353239623862356538396563653765 +35383862373263363438643735653366653664386330656565636636613735346431393539393564 +63636464306661623938316264346331646265353263343161313534313161343965626138383838 +62643937366266616434323634396430623138303366393632393234633166633030353030626665 +64346336376433626265626337336633376539376335326163616336636266376365386161383565 +61346239366239613961353862333530643363626538663434316662613632623730396137373435 +61633734366565623966633665613536316635323333366437323736346363313664353333636561 +37366562336438616130343261313939663036376339626463346536373063643634616234656130 +30393863663534633735373733623230303838613565636434376539353536323261626462366131 +36383261613136653236373065636562656135663038393861633131666266333935306636363035 +33393534653737306637313532386666656262613266626463653039356233356561626435343533 +30616664363039653936333938346233386537383062303732613263663962383639323131633564 +35343165356461366537663436333931363935656433613162323231656631643261356531353430 +64313363666138656538396135316632613162313465376536623266653466613961366261366434 +34633330326263303730616133313666663337346234663865306139393065303030393964653931 +39626235346133633561663133333935343332313738343262313066623465316630306464333361 +33343461366364356165623534336361306461383737373766623033353339313538653361636435 +65366232306230323837343361633766323431313032613866313130373639643866623535616135 +34663135613639313463336138393463303665313965386562663534323939346666353735333933 +62313266383835316364623733346262316664383533666236623037333132623631303338396562 +39333162313162393661643039353536323339333737646631633136346335303732656638376634 +66646131313764643164303132343439663531383764353332393230646135326432353934626437 +32646662356264346238363536616632313161363530656636316363643236313231363564313439 +37346332376466393061626334323436366635663338643232666434623761336433333964326135 +31616165663039626635636138653765386665346239656439326436646531363834313030633839 +61316563386233343237353038646361663461616130383463613934653065376365353562393362 +33366263363831616364353131643665613938663562336237396566373362633038623039383762 +30373334663234663638636635346238373464393837313065326636653563316438346332306563 +36646138313531663437653235633463623434343834363830343334366538363361353366666238 +61653564656461353963663961383963343133336331306436623935383837396462343337313163 +34613933373637323732613333316232323130623637663466303366633930666235633634323139 +62666566323464653934333466646663623533326530646532356332353238373330383337396465 +33326135323662613063646539636133366130303636396537316262306264643163326466653430 +65303866326638353463633662363537626234316264386563626365623536313334356334643466 +35663439663139333431656462356361396332313438396133356638356264323464363134393934 +32313662383339376534383232383634326635343338323964656462613334313961373836393366 +62663233343464643463383231383638333464313666633961663030656665383730316436616137 +33636664326664373864663462616539643935366631646666393966336533656166353534323835 +64353963643663343431393030373336326436363665343834633430653031353066656664373666 +37376332313031643831373130346636343839306537386566663339633738633763386631353537 +30393931636434623930353838303230613938663831376138663261326165333234353235313162 +63613538323261356564376436366462353935353531383861323232373138666132383634306462 +61613536356538646131353336396334343932343262663466396338316664353537333932616263 +66303737353638346161306537386633623264643734623832396239623530623934343465323864 +66376461636339656265643137643139313930626235663561653536646433623731396234653765 +39343036626662313837646638313430313962353137396633353430363263646533623965663561 +34353066656338313166353062393731353039663932393737666166656137363137363763316361 +63666265323135636538323234303433323966313338656339616236323966393238623865383037 +64396161613262643532393566663333653235386365366336626331396466633234633364386263 +39636432383365313430326431326464373631303732376366333532393037636264656163393366 +37383662316139333230363630343433303836333531313939303265613135343565626565613939 +61376238336531393435353561346539303231393266303764303732623466626330643136343464 +32663365373063363061633132313265373264313561623337636461353364633038366563376637 +33396435376333663066303964366633323836396332636230353862313437623535346432303263 +33343733656436373839336361616364386466633162386232303939353835613535333566366432 +39383339326165666464336263313732613735303430346234623566653237386564656431383832 +30326636626261656362623132333864393862663833303161326233653562623963666336316635 +66313262613366303761633132663666663032663164616666666534336335393638626135626534 +30383439313562396361663162333039376436373930306435333962333564666136373236623136 +66303830336233613466613530393066643366613866663931353565393731373335393637356461 +31323662326164326330356632393865663062633965613766353230646332616339663066656434 +32313937393430323232613466323236383362393964613866373863613034633331646435396465 +61323264383333343635356134313963363330306436663938386461623037653735626338366333 +39376233343562396436373135396362656461323665326638656432373231306465363332616465 +64643430636232333232313066386139356236613839666138313061343765663434643862356138 +30306663353132376663616165353832356263303437636531626365323664393334646331646130 +36363139633934333066626464316631306436306334626565373830643661393061333334343939 +61343861386266623032323263333333363564623637613539376338366465653733656631303064 +61376465336661343436393663343738626633383237663935323132326639633338336533323333 +36373539373731373466633838383561646663386439363736333434616364306663663361323631 +32373366356631613936316431663737616437353234333437303961303333353132623930343139 +62366466663766333662323962386161373736623336303362653163386562333435666239626465 +65643239303863366166326463353033363531636466316363363262323336366137663339623732 +37656562633431636632626336656166366231623030653362616238346266616464353532646664 +33663931636665323435343334646337306332333136356538366433353662343463333033373934 +65316232343831626431633966383234363531393032383439383130346263636339643639636264 +38666536326634336431346135666564343832336338323962343962323032633063323639376433 +61656531633531366663343466616466623533623035636366626463653734626562316136393537 +65616162656133393035623262346438663439356237643462396634383838333366653132363837 +32346436393438353364653264633561636635623833386232646335623539393930393736656666 +39383638373065663161323362313036626535306235663931393537316339343735326266353839 +63306565336264366663633161383933373430376535623438373939386463616336316135383564 +39313561626662623562313830383037353133393266333631323637326263396363336439366131 +33386431373534626135643337313838666635356361653966383465333062643638663830363830 +33613237323337303936363038393136383732323534626630663864616237666464303039313033 +31343861383863336136663764346132616265303532623263643937323362613636636166353062 +36666664646530343161353430323933343835663336363161396365626665303635346164396430 +66646166643833353437653062326137333164393766333836663939326238383435663964353665 +36373035373563616130386362643265643166333933613436346130353438343734623264663236 +32363234336162326530353539363532643366386162366330626630363534373363386537653734 +32643939633833643562666361663337346231303861663736386332663362326132383131383862 +34333466656233616561653865323635663932643566316161656661353936346264376633326132 +66343635623036643932383333396135353139303136383263376366393762323961653539633865 +63303337323565316337326330653731613962646165663030363265623362646437663939326230 +30313835343061613263376339323536356163616432653531376132386135343438333961376233 +38316261636239363564306432633630376262393336333862666533333539626437376164393539 +32386666376535656266383562623565623065363031363762646439313434303233333735396632 +62306365383662633139323066323630383936376339653637613764613161393537616161353366 +64653961633135636266396336386163616563653063653164363034643361383638313039336162 +63373134656139313761336139306465643233336265306361653932656637613530656439373834 +65373334626330666662383532303634336331633237396238636338383835326164323136643165 +37386464666261343536626164633235316662323164333865303265353563646138373566613739 +35346536313164616330393538356432623137333337356161303039303061383766343332363865 +34623831323364316630363061613533323061313333643639646339353963666636356537646564 +33633739636134383031653861646663383762663537376233393764616266306363613230373635 +35643862663466356134643435336332656361613437383536616163643133396335653937383065 +63616338303365356264613939663035643533333738633633343037303665353935663139623239 +65616635323866356663303064326664303534393430303362663361323064373630656366343964 +36663237636433343566326165383937333436373939646461656233666166656461346633306261 +61333739653233343230316361336130633564666339383234353036613036353064616265363931 +62383835323537336137373865316266353564303735363464303563623031356264326134653931 +36623563303134313731613038656632613734326464633066616636646435393462313365633364 +66353737666436343338303966313863376235623866356333353166653366363630393135613530 +33343938623039616231653038386362353534333738656232663661303365373363643138373961 +39323531306535663763316338383636376536626261373165356461646664376461643031303366 +35633434393836323839376534623362363863333562383733313134346261306365626266366237 +35663631386130343562313566653838303238353939313262326565386563366266386633376361 +61636365396237313332393363356662336439333930626230386630623238353562653435303233 +31333866306663383730316665313636353866343530623539333334616261303061343361396135 +62383663663338643730636161613739656132326538353639303763313330633962613637333534 +36656133386634343864393363616435623763316236653831663438633133326166386135623339 +35393865646263653532353366643061626433396138343431303238633736636334306666616265 +31393062303966326330373566663931646438306339363366623961393238623033623931353432 +37383337346334343538356237396535633032343164383030333235636465613932626464333630 +65396164316338326436303465663739666531313337353839363737383531343261313761356461 +35646236366663633662343565626635386462643764646265313133663464656165373563393438 +64313863373635656433373661623838383665363761376337393562663436333736356265633036 +62326132346235393737306539376265306332353134373437393138323535663633376261353038 +66633965376638346138333738613366366263393936653434653836666236386161623438343633 +37343835313266393762383463343530623964393433666139666638316262666463626230303630 +64333133353662316532653732316266323162663732383531613161363962363131363034303463 +61323561393835613635346238303639613037623035626666313437393262323265323634643262 +33353639306538616430326238313033306230386337643634303731653632346537623033386264 +31633866643037666336303134343931373362363936626565303937396335333064326133636432 +37623535333762356366666238646266613863646237633463353039343264323832373137663139 +38373961636466623136366636636565653264663733366630313537653033613132663562613339 +32313365383863323863343937333262633037393066383765383262333665323234663731613863 +33323236373066613533663338323462393231353539316165343361336663383737343032623736 +62356231613433346134383165636236353364396264613239613034656535346130366630393832 +32376138373466316238666530333561316638373133303933623730666135343262323836666533 +65633238653137306438636236363034353432346662636432363964376230613430636664653762 +33303262373961623963653130366661373438623239623135353162626532643064353965346331 +61393832323066333936333638316335336431653838346265643232393836353038646562333336 +33666237323439373338313263613766386434346636356663353836643031303462366431346233 +62313138643434636539386534353936323931656438376138616431643339663637663536326236 +64306264346265376462636333343165373966623864633464376635376566303234613830353561 +62323235386230306161376664316132626139623561373339306462653839616433363738333832 +39633837346436393165343863323138626636396631626337653631373466386238396665623338 +38666164613061623734393436646130653663323064623335393032343432646334303764333662 +36623234613265333465383538333639613336336263623965373161653633393463373061663765 +66346633636463323464346434653338633431636130333734313538343266633739626237633936 +65623664386665393839626565663765613135343037323663303336323637353134373864616639 +38613664396231346365383164343034633331643530623238396462653436656637366264376465 +39383538643631386164363039326136626563363330306431333566333231613131326332303366 +38396263343461646137393665353236363865616535353665613933363763643265393837333036 +35313664376364326231353337633930643735626335376136333935376233356235316663316639 +63313137656133373236326533326138646361643831613236376433323532636264656264356163 +38363234323931663934643964663133303865643730333135363762363139663735633534333139 +66393061646166666666666265646235346563393733653537613630643839616432616363313464 +35326239313730313237343666386138313438303136363263306462386333343463306536373666 +66653164396461663232653061313063623931313338376231316531326233623633353137623736 +39633733646435353639313063613030383466396235356361303732383531373835396430366432 +39323138333761373065636563326562633437323233633466663965353330356661316631616339 +35356131383231313833653732356137663639366439386139333438623334656138333737313461 +38373830353032643738386438316564363565613232383561613861646566636239663231323763 +65306161313562373133623339386165386664323639376136626666363531303735396337623830 +36393139346239353033303664333566306233666366643033633636656631626533643538326664 +31623666373461663333326339643964616264623163396435323934313132663733373230393731 +32316534366463326130313531316137633637333539366165633162336437656535656634396434 +39613037373665666234323935393165653539373130383561633030386138646530363263326666 +38323139393365326431383065383033313335353236623031626365383562636231636633363839 +31323835666162633462663136393439656632663336383531633865306463636133303961303863 +33353031623333336464663065613731613333313539383431636638623330303461386430396636 +61383636633430613037386164623238326630616234616438313962333433336564393066336461 +31666431306435343536386238313366336663613061633533613339353264656135643165313737 +61323230353063313166323531626230386236376136353163356236356236363364366365636432 +39383532383333663934653666333062396233313664363165313330313230383965316437373231 +64393835323333666263336333623238313334313863663038393563326633353232373765346361 +39356337643836306135346266333233393463653539636534323338613633313537653664336264 +36396332656532613738313761656337386561346237613661323863363134373965616239313632 +61366265393438373738386230663863323537306336366562363363366632653862363632383464 +62616137373037346664663339346331393233386630343730643965396235633736383064613462 +31336261653133333762313036306338353261626464313364333064376436393662393738363862 +36303365336661616362363831366334353834666238653561613162653162633061623631623936 +38346131366134333066353961346636356535356336343565366431303262323730633639663733 +37653833633462303235623835396239333532303066373433336132366635626364353439633530 +36303932363461306239396634666163373837363836366239323162633136636136356465383231 +33306130626666326463393131383935323465313031393631643136303830323035333432393039 +66653434666638623263643839346564393464616438663833376536343664633637313530623538 +39633934353437393437653663396633306461613839386134613432636636353266643830666234 +36636535333930353962633638633130393265626638633238376335383466393065323831313232 +36626537613432633534336663303735313339363230326234396439633136356531613135633664 +64643463356430333732396337313931653333333231323762373234363063616631393766373163 +63343332393934306534393538643030373562393430363838636139313237653732336337323232 +66313264656166653238643539333464333737633866363062666330366630323231323039393138 +65356534633362663134393333656337646435353231343166623436356234376232656466633736 +32626264663532373863333966333039623239326436373861643139626438306632353735316135 +33393434616364393330356138653233393264316539623563323462356139653534656638306434 +31316262393437383364626236306331613732626435343336366639613661663035393835376565 +38343835383231333037643230323933333062333936636666653562623831306563373138393733 +38636133346533633032663732626439626562363139316331366262626238616561366638626435 +66346334663131383261376266333438666530313037306365663062666533333636326230313061 +62393561353739373738643262376538346661303862393433363462613731303437303266373965 +39663036633438363630306266626633636166653338326139343466623134663932336435393966 +37653065653135326138623535353865646666343534343833366161343238346133613565366233 +36666630656438396131636638356437333933623532303734336365346230323534306139613031 +38326464663264643263376434356265363436386536356434386435663365616638643361363930 +33313835353032323434383435663136653431636461663537373030663733626639333832323930 +64323233383064326632646566666431613837356139303961623733613139623339313161396136 +37656135346566613136303236613239626630346165663831653735333330333934313839626433 +34613162653864396565636462626632346635383662653337616639626531653838353066643135 +32363465323930646630663635376464363232613030396230336230343361356366646536626134 +35393435656637666363663230623832666234346233303832366633346638363532336361336239 +37353337303264633164396264383630343731396336613432316631306364666163343236623566 +37316662346137323136373162373762323031326339386336366138363266313731626334393431 +36653663613839636564663430306330353164376531386233343561383337626637613631316330 +34333437333565393430623535656361353266386235353830336666343135316362636338613562 +62323935633735386166666539303266346135313832303932376139376563626638623362633635 +37313834623165316563393630343433353266653335383161386238376531643863646136633738 +37333063336162313362613231613638303266613861393461376663393132656162373739633236 +63653035386634616539636463363130326332616232373265636331383766383066343137656537 +30663336333333363235376430346162373261613066346137633662376135333130316463346530 +30373230303764396133396336363662303134326266363336336536353635353230346264623066 +32643461613165393866616635663239393862373539633765306430623163376637303935393839 +34343666383561613435373430356361373439643564326337303766376434613561363761373261 +39306461333066643538663965363334656632363132343663626434663835363338396537653863 +35613763343837393938623063653936636366373130366462646361343131353731373632623664 +66333765633432373264303031323731356233633537636539653833323131613332326530366630 +34623830396436323765613036313233643566333134666434336662616363316565666634656136 +35336632353137646464396437363135356262656635653739613832353533386331346231643964 +36386631343037366335633737653735356462666237396534383166663631313966373366316333 +39396134336331353636653133393632323139623865353863643831336266363237376263393962 +61326263343838626537313138346537366539656165376162643561386531663433623730316338 +37393564333061346436623566336664376465616631653762353236313162646531323763396434 +38376362616565363961356364636338653366313431356161303337623035616439346631356361 +30306236646536666637336633633837313330653165623532613036336662323233343736303665 +38316139373838373332623563323537366131663566373336353438666664393761646332333563 +63646664323838653933333937613265353530336132343763313134343731623461313632353934 +35353736633965626530356262623330373661636265353631643461623665343331366638666263 +35666336633562343337363537623939353038313830626666333562613165613066303738326666 +31633336383831396635326637373661316163353865373431383338613361333566326130333231 +66653236663333373938626435623266326237313165666266396363663833306139613965386432 +37373366336633643964653935363565336332613364316433633433646133306135653730393861 +64393037653233623333386239636233353764346535386564353463633036333732356633613631 +39333763613466636461346633326235666431353335303163386665343763623266343636396335 +32396265356266633830333132663632623031306662656363333166616362343763396364646436 +32333063333130336131323239303735313836646366396665346133316166643739366632666437 +34663863623133343861616261393265336365376139386237643362616535656635353335616335 +39393439333661373333366439323861363534333461316161663166326631663132363033356561 +64613736626230343832336435366339666634326230636137353764396338636131613838653338 +65336639363138643534356262396132373835383335363539643533643833343136613536336561 +35656330633063653337303336343266373063323161666566343165666438633263613333626236 +63666265316136313837313537333761643238356139356433636633343664396162623565646666 +32343036623538306238633464383830656332326234633030303233643461653934653962653333 +34633763653033353466336637386136366336306132653735393963303832313630653137383833 +66383735366330626431376563373235353733306166633333323138393736653539316261626164 +33383238336662383162666266656631663937663062303764353763633332626338333432366262 +36376632313332333437336261376631386339633938393464393938326632323764363864656330 +61306134626564616332623662386330646662373265363235366334323363323739663838393863 +63313031383538313563616634396264313533653536326465643034313663363631326666346231 +38396561653861373862656231313930323137303336663236393136616134313034313631323334 +61376131363735623366633062623861353731376463346138636462366534346331626238643537 +64353933373761386238396334333361613230353633653432343964333931373738316533353530 +30343137396366633433646330333665633534613332336539383030306562623034323130666330 +66656463633630316565653161663339396434666463313464623666316233343139613762616262 +34663735343933306665373664346638306536363863616665376237336664643636396231393061 +61393032346236666631366432376434356135656539303761343639376533383934303762323637 +32626663633536346236303036323364643865666562633265666634623336656134643239343464 +36336562313836313965323630613032623334623931366338363033343333346636643466356166 +37656666643639363631643165666530393431353732653437353532643735376166663534323838 +35336137646530373061303065396639393938363963613061393235313037653163353362326636 +66643137343133346631613237313636356264643562396535326537353164303239383261363031 +36326139333236306564613761653661376530376236643131633135313934353338656264653061 +32343535383032323838343934613261653036303939346131653335316462323230623136323132 +62356435303239393332316261383432336234613362636361663461643530313061396638343262 +31613466393964316663383166636463383032313065623637343538336439663633653466303138 +64326136376635306431323365633038666631323833373330343230343162373761623730643166 +33306366623533373031383762356136333962393835316661346533303730323238363833343764 +36656637363365303931386235386434353833636463623836333032393666316131616365616139 +35326638666336633366316634376663663664663335393537306166323936306231646432366335 +65316635633436376339663666383633396236373734363062646531326330303038623339346263 +66323461393937303134633863393664666566373764653934663730363431323634323863303639 +32373633663966653565666363626130346134613439383231623339363865666566343837656561 +33666537323736646363653935383265636635316364333264613066386336396262636639363932 +37663262373037633135303032303734633836313135303731326537383536393863336462636164 +62666237366664653064366165333236663662616230373362656438303962383237646364343936 +36393235346335303665393136633330613563646331663732626639613330663231306165636465 +36393733363934316137396132363033313031313835613934666366363834353161306334313962 +36393932313264653838383438646537393337613734326534373466393037336664663861653063 +64373633633033383964363061653666393332636662343838616564653834326332663139323564 +38313466666664316465323261353661333137326266313866306163386462343539326132323764 +31653833353131313366363166633163346161376561656164373632623335383466373231653662 +34383235646636613930656265623838343531333733663437313461383262316564373336386165 +62336639643938316136363531316262626634636465646561646133303233356336613035383864 +62316439643664346636326539336632373861636330633039333563363236393165306237663132 +64363265643535396537313161343963313833306332633062616563653237623439346639636231 +37636137623961393465653833653139323037663435333763623061653566646535353763393035 +38376366643330393034326235633765383337333135623261313832393134373236366465616435 +66376465633737346561363464663433303166636563656233336434396432633139616138363361 +64323564343034636337313535333765333530633830646334323932373034313333646538663133 +36663530373139666132623333333266346233313338383539303033306366666233386638363532 +30643132356138666438303233653264643165356334653236616139373331353938313332373566 +34353332333063643333336537326233386263356230393165323534343536656164653433376661 +35373337393336386163393436653537386337306466353365313062396335613038613565343032 +37396239353263396439653166613333306162336564326666666439386364633864613162666566 +36366564353036633036323761303666366335643233366465316164333062393937333635326636 +34653639303932393534366262666364306630313265353433346631633531393063613935303432 +65393966656166626338633532333966396233613839623731336566383462323634653930333833 +64613538643465653666643830386634376664396535393733303533316638343733666161626333 +33363635363464623362386330376162633838333566303936303366306364613632306263663339 +31376539326534653531646231646637326536333761633435663235313264613436323538623934 +30346465323638373538623761623338623063333937363161353266306533303633323763303532 +3862353738313031326238323866386237623735643630303639 diff --git a/dev-tools/ansible/inventories/dev/host_vars/dev-server/vault.yml b/dev-tools/ansible/inventories/dev/host_vars/dev-server/vault.yml new file mode 100644 index 0000000000..df6ec38a85 --- /dev/null +++ b/dev-tools/ansible/inventories/dev/host_vars/dev-server/vault.yml @@ -0,0 +1,10 @@ +$ANSIBLE_VAULT;1.1;AES256 +35393463623034396661646365323430386238323037643233363431643463353937653038366266 +3335346534353634343632306163636566653666363562650a363839616365623364366434666537 +37333436376430346630613033623163633637626433313763666533306633643065333431616235 +6463343263373464320a633537626432353533623139623438363834353464366335633163303162 +64623433363938633131613936333037646562623233353833613734623934343731656130613266 +35373733663962373462366234663137666634636163323733623830613164616432316266356366 +31633231316439633862666431626235353530623365623963636161386232366336303138336564 +33363064646637336232383232343261333932393031653364656565326365646136636531323639 +66626539636637626336313365333034363834393636303866333534633761656631 diff --git a/dev-tools/ansible/inventories/dev/hosts b/dev-tools/ansible/inventories/dev/hosts new file mode 100644 index 0000000000..b1f22a5f4e --- /dev/null +++ b/dev-tools/ansible/inventories/dev/hosts @@ -0,0 +1,14 @@ +[airavata_servers] +dev-server + +[keycloak] +dev-server + +[zookeeper] +dev-server + +[rabbitmq] +dev-server + +[database] +dev-server diff --git a/dev-tools/ansible/inventories/staging/group_vars/all/vars.yml b/dev-tools/ansible/inventories/staging/group_vars/all/vars.yml new file mode 100644 index 0000000000..d1cf61c23a --- /dev/null +++ b/dev-tools/ansible/inventories/staging/group_vars/all/vars.yml @@ -0,0 +1,138 @@ +--- +# Non-sensitive configuration variables for staging environment + +# Airavata version and build settings +airavata_version: "0.21-SNAPSHOT" +git_branch: "master" +airavata_git_repo: "https://github.com/apache/airavata.git" +airavata_source_dir: "/home/{{ deploy_user }}/airavata-src" +deployment_dir: "/home/{{ deploy_user }}/airavata-deployment" + +# Maven version (should match common role's apache_maven_version) +apache_maven_version: "apache-maven-3.9.11" + +# Deployment user +deploy_user: "airavata" + +# System user/group (used by env_setup and other roles) +user: "{{ deploy_user }}" +group: "{{ deploy_user }}" +user_home: "/home/{{ user }}" + +# Service ports +api_server_port: 8930 +api_server_tls_port: 9930 +profile_service_port: 8962 +registry_port: 8970 +registry_server_port: 8970 +sharing_registry_port: 7878 +cred_store_port: 8960 +agent_service_port: 18880 +research_service_port: 18899 +file_server_port: 8050 +restproxy_port: 8082 + +# Database names +app_catalog: "app_catalog" +exp_catalog: "experiment_catalog" +replica_catalog: "replica_catalog" +workflow_catalog: "workflow_catalog" +sharing_catalog: "sharing_catalog" +credential_store: "credential_store" +profile_service: "profile_service" +research_catalog: "research_catalog" + +# Database drivers +registry_jdbc_driver: "org.mariadb.jdbc.Driver" +appcatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +replicacatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +workflowcatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +sharingcatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +profile_service_jdbc_driver: "org.mariadb.jdbc.Driver" +credential_store_jdbc_driver: "org.mariadb.jdbc.Driver" + +# Database users +registry_jdbc_user: "{{ db_user | default('airavata') }}" +appcatalog_jdbc_user: "{{ db_user | default('airavata') }}" +replicacatalog_jdbc_user: "{{ db_user | default('airavata') }}" +workflowcatalog_jdbc_user: "{{ db_user | default('airavata') }}" +sharingcatalog_jdbc_user: "{{ db_user | default('airavata') }}" +profile_service_jdbc_user: "{{ db_user | default('airavata') }}" +credential_store_jdbc_user: "{{ db_user | default('airavata') }}" + +# Paths +local_data_location: "/home/{{ deploy_user }}/temp-storage" +file_server_storage_location: "/home/{{ deploy_user }}/temp-storage" +agent_service_storage_path: "/var/www/portals/gateway-user-data" + +# Keystore file name +vault_keystore_file: "airavata.sym.p12" + +# Email monitor settings +email_based_monitor_host: "imap.gmail.com" +email_based_monitor_store_protocol: "imaps" +email_based_monitor_folder_name: "INBOX" +email_expiration_minutes: 60 +email_based_monitoring_period: 10000 + +# Kafka and RabbitMQ +kafka_broker_url: "localhost:9092" +restproxy_broker_url: "localhost:9092" +rabbitmq_port: 5672 +rabbitmq_status_exchange_name: "status_exchange" +rabbitmq_process_exchange_name: "process_exchange" +rabbitmq_experiment_exchange_name: "experiment_exchange" +experiment_launch_queue: "experiment_launch" + +# Zookeeper connection +zookeeper_client_port: 2181 +zookeeper_connection: "localhost:{{ zookeeper_client_port }}" +embedded_zk: false +# Zookeeper AdminServer port (default is 8081 to avoid conflict with Keycloak on 8080) +# Can be overridden here if needed +# zookeeper_admin_server_port: 8081 + +# Database server IP (for multi-host setup) +db_server: "149.165.155.21" +db_server_port: "3306" + +# API Server public hostname (for HAProxy SSL termination) +api_server_public_hostname: "api.staging.cybershuttle.org" + +# API Server connection settings for Django +api_server_host: "{{ api_server_public_hostname }}" +api_secured: true + +api_server_bind_host: "0.0.0.0" +orchestrator_bind_host: "0.0.0.0" +registry_bind_host: "0.0.0.0" +sharing_registry_bind_host: "0.0.0.0" +cred_store_server_bind_host: "0.0.0.0" +profile_service_bind_host: "0.0.0.0" + +# TLS Configuration +tls_enable: false + +# Let's Encrypt email for certificate notifications +letsencrypt_email: "admin@cybershuttle.org" + +# Keycloak virtual host hostname (for Let's Encrypt certificate) +keycloak_vhost_servername: "auth.staging.cybershuttle.org" + +# Firewall subnets (for allowing access to services) +# These can be restricted to specific IP ranges for security +# For test environment, using 0.0.0.0/0 to allow all (adjust for production) +zk_subnets: + - "0.0.0.0/0" +kafka_subnets: + - "0.0.0.0/0" +rabbitmq_subnets: + - "0.0.0.0/0" +db_subnets: + - "0.0.0.0/0" + +# Other non-sensitive configuration +enable_sharing: true +enable_validation: true +enable_realtime_monitor: true +job_notification_enable: true diff --git a/dev-tools/ansible/inventories/staging/group_vars/all/vault.yml b/dev-tools/ansible/inventories/staging/group_vars/all/vault.yml new file mode 100644 index 0000000000..1854598609 --- /dev/null +++ b/dev-tools/ansible/inventories/staging/group_vars/all/vault.yml @@ -0,0 +1,534 @@ +$ANSIBLE_VAULT;1.1;AES256 +32623336333736396466346462303332323530646337633462306230386566313465623265663733 +3763626434663636666530383339633436623237333431370a663837646233613661373932316131 +66643239336438326363616331323333333331666538363563396336383563623538353735613962 +3231313234663635340a643837346439323066653934306235373164613765626639303039383132 +62306334313532643734643936643265613161663063363631623739373239393032626432636164 +32613934653936366336326230383237626661373138616465373737303039393538333538616234 +33366634653031653661306232393434363164326462373564346561393535353838663062343864 +65666637303632366363343231343735646561663534326334303461616533646533656331633866 +61326264623465653331376363356262323065353934386366633364653061633336303034316235 +61363965616637383930393638336632643133353638343135313331306431323531346431373861 +33343530666435613338666561363061393238326138663563646163303730633137633236396638 +65336135613564633663623062313633353564623337356566363262323437343266626530636531 +38656233396563333433636362393765363435303334663062336363313761626362313064636463 +35663431383135643731373461333732373731666364323863653439623862366661366666356265 +34346431666639323566623137376232633165366434326264343033613831333262396431653330 +39306539616238323339643830626233393138653562613435383430376263333135386134366665 +37373134613736373336303431336238393762353932393136386131383030396538306536363662 +37646362663536323065343230613635653739646538313432383961626562636331313032623432 +61303166393034396432363339383036303332353938376664636566336466316638303839353337 +62353335306361383138663764616638336531643039616466346330356234666166623861636664 +33663334323133656134626331653331383030383837326363396566313439666665386633623266 +33326163626337653537376636346638313565343730383066356536656132663866616664323330 +65663634646238623335633662666532636336626334636535626537363430316565373631353632 +31333633326361366232623130343366633861363634666261623863663264356163643631346330 +37376234383261333136613161383666356538643538336564623631393930333761386435363164 +32323931616331623630353063356666306138386665623731323666303338316231636437626535 +36653066306238366436656534636632393538373131643061383362393365663463333137653162 +63323733333539393130376563303031623966646361393762666231303534353530653938633065 +61656130343837393030363365383664323465643036623666343530343562323436666663346164 +37333137636133303035333462663438366136343634313930656433393139396232633733623664 +33653133366436663637663961366136306239633365653532656334386333346532663362376464 +35366531323231363334326435346239333862656537363838633736643865623864623734663562 +37386131303231356437666363343933353635663836396662303436633838386566343730613834 +36343639313830353035646432396333386130323034613337373363316665656631623164363261 +62653562323361396330373534656663306261386337633865373731636535336436333562643864 +32323136656664353264333464626539303834363361353131626238643862386565363634663762 +63653666386131313231666464393237383539636161393363383439656331366262366464343463 +63383637323936353131636631303732653036633063303366376561333435313139636662363531 +36346334363130633162333533343138656132623037316336633262663830346634333231323132 +39313933633237343837376438666535353734333334386230623162396435663732396561643865 +35393830383066313537636136386531363834343436323839346134383137613430626136386365 +32353839386666663935323462323732336430333965343230383636396164363965663937653734 +33396365313665663333623832356633316266306433643461343563396334313866333862656635 +64336430363739313834333139656561643030393962313634663362363838656333653962306234 +38643036303865313932353731346363346666313634326238633039373765323531353765613932 +64373435323766633261303563303266633561363161343235303336666262306533346139333961 +63633662333862316233393665376565313739383633623534303935356430663266303165383038 +31643934633434346331386539613662346630666135383665373037363439613062633866336434 +35356233383238623732646661356339396239356634313164666435313037373837666664383730 +63653365346566653132653636313532373762373932373930663463363038346666613061643965 +38643236343866613939633065373364333565393831333736326262633436323036613265643132 +36653538383633383739623637306464623234313934363330623164396564643562383563373233 +33636637333338616535343933663461323639393061306239343234643362636364646430616464 +66616630386563653532663236343666326566626631363737316338663538366462656638396137 +33623263323262663337613761333136306335663065373533396563303538613262333764313033 +32646337643666363332323039333933663539643363376563623761333964633033386531326635 +65376532613137356633646638303538366134386535666135326636646261366632623532623531 +30613933323631336363373763316335343639306336343534376638396561393936346361343635 +39343962386531303463343537333536386230663333646537383230373035323939663537656432 +37653162646636613930303936313232356438343864356165336338346465656337316231623430 +65643562623534333961663331653766303933326463616266626165663639343537633235373762 +36393563616663353662353538346465323864663137346630333135653036333738326466373132 +64383065643963386139393834643334373730343139656230383865306664353130663733613338 +36356566363466363932663734316437326230336665313765656564373736366663616135333733 +39633838306461303837383638643037393534373231346630646135353236303839346661326165 +39386266643435386266363735646164616664636563616665613535343465303437316462653137 +39643334643263616363353435623531343737623163356431326365643039656362316565323133 +34623130363335623030326265353664343532663237366230316234616632333166326164303963 +32306435613863386663333930323832353536636134643434303833663165326137663730623432 +31643163636465373034326335666430393238373136613636373237306136366233386237643732 +38333835363861396438346663316263653230663731643036626236386632303932373861303539 +33336336383832313532656136623462386233323330303363613639653135386534316261306138 +38316239373462373365643263616232323063633637613463323461333936643932366265353032 +32366261323133343538646433396136313664313032316438613932356262623832613535396439 +37616638336363396639633937333039623263363161663963626538333333306235653361653237 +35613962633933646261633664393964653066326139663532616238633636653932653666373835 +32373264386536356464386136353839343838303164363165643038633333623236386366353466 +34656333303361333637383162646136356432353134626363653435303637323437373734346139 +39376462313930336166666261396436373938356536376665626663616631623235346366353161 +34396561313530366630343661623766353032323933346236633434366531616234303261373332 +61656466653136323335616166356261333761616133303537643739356364383638666333663639 +31653131353766383039306338323463343735613161623337376236633335353832333664323131 +34346535326636376239616530313662363739666631363638656430646237323635366632633834 +65316363303332346161646432323862393636666333353535363362313964366462636338393135 +31663265343531376330353730376164313136306562653435336537636532373161313935666636 +39333533386165626161666137306238343566313562373333366433313065333037343031393865 +63656161303366623430373864656631373230376131353530303565323066636463646665303636 +34383530383331323562663939316530666133373762656135333833663836353565616431666233 +36643061323463333533396337636637393062396436386539313031313364363766623937643663 +64326139333831336661663132356562346539343431363038373438333565643737663735323339 +35656231393563616630376136346161383937313038343633666561313862313931336566616563 +34353530396538393133346330306662306138393633353065326566373634336532353935383965 +34323664326466646361613234663339316463323461653863613737646434316239353361333766 +36623034336334303035653263303039373233373936616631363162376634626637393266306537 +63386362326364326566386165363833396665306132386166623432633061366533663366626336 +35333765663439623430613863366639613131383062383738343831633031343464653937636466 +38346665336634663434313034613765393463353231653832656665363436306566353865393066 +31373437303734643766336434386635656162396464616431633566633165373733323265623437 +61653263393630313361386534393039623833636166623434616238356361333737643134396535 +33306332383238623830663932613136326563316132356633613463316362326165313738356430 +38666265363534346564333562366136353132326232613130303562653434643837363339613764 +34303434666465333731663461646236656331346239633434343638346339393763623864353332 +30303665323935613235643239613463613737383162373662666332636631383436306530393136 +63393864643466373534303861626165303265376235346634396664316139303563353735666536 +37653265353064313434396339373962633030353961376333306665633133333163656235643832 +36613164666365393966353937323731373665303331373864353561656263393431363464643730 +65666665656466633437636362633436383933343765383766616233653234386662326333343931 +64623036643036653134326538363465646463633065316661613833393264356431353731626531 +37356364663165386333373730366161643965353964666331636333656165393539643332356336 +31306561653239656634323332333333323365666666643761313137386662353365623538643661 +31333634343535366161353963366435633438366637353564313730616163323236363361383632 +35663335393534383765383538363565396262643831343336623561303236313466336133646565 +62333436303265323861306135633463653434323864636464393061663461646236626632643663 +31663066373135616137353961663632396539333134363537613163313638653965663031313531 +31393634346265376564613532343031386563666432383130303237366562623733616338356534 +61396430313133393337643232353730373932383764393038613535646563616337353936646131 +61343137363036353932636438353034356238636331623637323963323637323165633035303165 +39383463626538653262656264666638653138613861313738643664643062383639663465316238 +61383037386338396663366664363965623639306462386466316564613837613231336532393165 +34616637356434626263303432386662636134623937383839343961306530363866313765336135 +62663962656330663932306261393139653536333435376438383532326330643564353037643061 +63643138656334303637396332313236386561383439653134636233393433363363373934613763 +65376635646563393463616135663265613539643566353366626562633265663638623339643331 +35373966333533333431343734626530663365396132383638363436353931363338626265616436 +62666432346534376638306431646230333538623433336466626130306136373639636538393538 +39633963356133323366383531613037333834633663386465313938343061636532346432366438 +39623863366330383562383231363434656634303830393761643036666436643937363837383865 +63373138356431313635393134636439623239616138353430373030623733656437383531623533 +32626131333139363138333864363337633964613130613963636136316566386138613764363465 +66366436636439613036633935363333643935636132343134383235633866373137376230346339 +34643037396433346431636133343564666534653634303161356538303230313665306666333635 +66303062336534356162623838353932386434346263313763633734393830306235633830373935 +33343337653266323437636162346338326139636436646162316639306165636266363863626430 +61336231363834306538383166306434333461653833633538316438303836663864333830393735 +31326335336131373862663763396630333037373132613936633161396564616365663635343266 +30316436373230616431366437316532623836383239383865343931333330613435393761306562 +30316132323336653234376335383937303266653233376363373536323332633132623430663964 +37323637663532656134646237363162343235306266373537653165623631613262663735653865 +64636637326639313964366539326462656636313733616138353536646539636162326136336263 +61393736396664353033623861343066646564616330313738303637326138386131656538616163 +31663331356333303036386131656337626537386162643064636435653963623631306336663666 +38373130633130666564326239306666386137663438333264306137323161653935313735626131 +33356534383137633631353732383537653265623034333336636634326539373633373365616364 +66663736303633343035656334633139326138663634333834303631326533393135363331666531 +31376165633530306162656363333038616664386538643162313733363538626266393437323361 +61363161356439616666633163313037303731646237653533613330393765383564653539366637 +37623134306664316165646365313532316437336139653137653463646361623333356661316338 +33626564353339353730633363303963383632376136623534366531333634643062323332366337 +62346234636432613465336330383430646462623933666465323233393638326137373166313266 +64343066356166643735623939363437336565336661643735326666363865323766616339316165 +36396432383362393132636265323934613937343566303035666637356464313339323138666235 +33383631323363656133613939633336336139313234646631643665653665616264656163393833 +64306331643630623633643438653739643637613730316534323833366634623839653939373062 +33363564636461396537656165303636626137623931313733366463323232343362653334353164 +66346436656366326132343466333966643961376162326235313732646161663765626636376631 +37306433363437353030313438626466663035643137653465356563333230613139323934633331 +30373639366531616332636166363565626631613866616538626239656533303835643262363861 +30346266363131353565313965363431633162343663663132323035356630663864623331366163 +32386332653861623336653866666135396236386663366635323366653638316637656166316439 +62343835363263653964346166333931343262346138623732396339336333336335393735366632 +61303138306539366431306665653433376230636434323032366334366332613438383539303333 +30636466636264663239663934333265353762383565386162626464663766616138626461343862 +65663037373234666338666139316366333934323663376637623462623564633537653039376435 +33366231343564336530666661663232376332353062616565346638303832663332316332373766 +39646434393731383035373931646563643633373130616665626137353838336539613838623139 +64336439613537356234303764616131653865356362396536303936313933633363306138373536 +62363932613561383635626139336266356536633762383732656365383565386233626666336531 +31636535653037636462383930353062373235366533663633363830326565653365336231646162 +66653663343230386435656235373631656131623234343536336266623964643466323561373237 +38383033386231343438373635623563636265306161653430656566623262353838653162373462 +62663839303333633866366561613231643732336664393962663232336430386433356161356130 +34353735373263663465316166613761643734336234613164313265326262303635306530336561 +31336435663233326434346362666236393432383963313534333763623730646135633763643732 +32366564386333333339366431636336346332386638353761653262626439316230656432363033 +32363738316137376331383830313264336531363736616463316439316332353963326232393265 +38346133616536656332353665366430386266323235393663373565616461656337303466653135 +31373262343035656664656462613531386430383564333764396635393861376433356361643365 +30363130303763653839343338316536383530626335363261383537663262313037616535303130 +36303633343564663763333439633733646439623934623664303932313435353437313363396334 +30353062313734373937613663663536383061313036313830346437356265616539663634376335 +37663231353066383530373332396332393266613233306530663839386161383264326164636162 +63656336643933363934616663303161363038333637636461616564306461363932343432303930 +63633939363962623437623131373537356334633630356330396665313235333164393331626562 +31306432393662393037313935663435616335323139343931366563396162666135303665343833 +36313564646438343834326435376161623135376134633937376461653735656435396662363331 +38333565343234613561633138333061333435313761386639303036633066633361386263373266 +38316136613463666162346365313836653634623832633535306161303665663761336662303130 +66356332303037323732323131353338653331353733623564623765336332353330383233393466 +65386332383166333434643131303730636663333063366165613537353566633430336133613562 +65626563653734366264386330666233353664663532633034386562306132303166643161346562 +39353738636265303161353132393633383537363832653036633933356533656235383166376234 +61373332303866663964363661663938303833656464343861666633666231386333396465656436 +34623366323630333233663339346463653037616263323436633433373738633035626536646432 +35343533356237623463346266623932643932653866346133326136643366353163666232343939 +65643535653435646431393535393532316239343839303861303165393539333463653962333237 +64393934386130633364366661363034393338656462363666636133356164636137313334343262 +39303166343361663064303365303266386637616236386438623966333333333463353731326261 +62383137316166396439393439303230346632393132633736343464643632653661316563373833 +34303535663964656334383763343436326661326663326137633630303066306435343861373438 +34656334306166633232363664313031613536353337663538616338396566313031656231303731 +36333934636433383538663061323030656231303565353565633934323066356266633961366530 +35343935396438623836333533326332623339363739633933656264616130643330633530336438 +64623061323062316364363231333562326166666439386662373033323738343632343966303535 +37393632613962383235366665356464326264363666346232383533366163353364316134636563 +61343163333966636332373536303961383931323131396332373863353363633864343966383265 +35346331313537353338633164353633333361636230333266633838346365373032303935623738 +33323265363366373736663263643735653434306364323961393738643436636238646362333636 +61386262363163656438363164363835393538303438363736653731366161353535333830653466 +33323831383138336164373337623861333533333134646135666639393730636261316561393931 +36393832656463343137626336346333633730313037366533653930636365353033353563396630 +31613862306337356532333532323737653836313966373361396465613439373263613132373766 +62616162643633396461666135623364633631303639383633363163636135613461343061393962 +37343762623034643864363533386430393965396534323464623033356633393463323433613731 +63376630666630643334383135346233333363616330346630636566336465386166663132656461 +34383162333636616261656466363133663535346230333838663339663164383732356135393263 +35376161393432613962323964363266323266313738353430656438666365653235346435376137 +38376530303036303230373333316564326632343561373066613339346266356237326632386430 +64343065653336303439323432323961373761663038663435346137313332376438303935343161 +66626666396636366364663464626632346636613539333136656563663364376531373361376663 +39316437663831313238376466356564383830363836323765373162643565663231303735343434 +32346161393564356164616536313866353039663630643334346537346261633531643334656535 +34336531363465663839636138306637373638336533663038306131393066353838393066336666 +30663561303931643166656136393263386532393937666161663432313763326331666634616130 +31353762313066326236396338303162613230613365666134646162303864633463303737623939 +32623863626137643164363135386665643730326161383930626638353738633131613038323361 +32646432323137393065636466336531663938646462336131643338323339303839323434313631 +32323563653035386439613134386464626562326166613035333665346266373434373565356339 +62376565316235653431616432383435383638326365663765643765366233646333643337303532 +33636139383064666664356535616231353163303263616662303636383264316236613463303439 +39383736373538303562626331353931323463653231646433636338383338666337363461363461 +64623034656564643862303132303966376363306266366666616239613039653235643861356338 +38643638353733616338643738333139663763313939653865383438646338393662313639616636 +65333061616264353436396336303163373366353338613663613863386464623931323661356631 +62613534313361303237393963316564653338303963373861666634623037643432623336623862 +30353935366234393933376461353838336262643733643231393766303334643464626366386536 +30656461366566336435633265333831636163363365383365623230386337343335356336653738 +37353631633332663636663138636563313966636232303131363661386464386330666262326163 +61316535616333373334613862633962623661646237633632613336656262383231366532636433 +66616435343832643235383636356664336230393737323766336537333938616633313263326437 +34373731346663376231623830643631633333653331656135666564653433626466326235613862 +35306231636237663564396138656362313865656161646437373431666439336634326465386239 +39636662313165336433653334633132313066623534326337613165656431323662643936353831 +37313433636539323932633835353362383538333764363236303239313032323931393061653566 +37626333383931313964336466363261643465663731336266313937343066386636633431303330 +33623263343164363339373132363266313264303361303034366563653931666163303833646538 +36353062643835663234333432316564656338363035626662336637656532316637313738303831 +31646564393463396534323536613565373338386661333464633436383861363965643435666133 +32333563396430363766383264383065636536633637613062616463383831363664303566616132 +39346231373833306265343532333063383561366161363037373035646166383131646365356232 +61393236653462396435316230646234356531663838633033633361373737373865343966356635 +63383261353062396237643030353934393239376631623764303062333061306134653930663738 +61333365306565353638383830653738313632356436353435306362393537663761353130633137 +36616361393263333734323565363763363331316362666239303332313430346639636435306466 +61653163306433346535333836663930323962613939323533666463346230343563326162653364 +65326662326335633534383063353166316435326331316132353534643435383137323465363339 +38376265386336373464623163643164363432613361626430336565656164633063643036343664 +64393466323531323333656432333831363038656463633933376236303431343936616630313166 +39363961376164656131353432343730326562376662336665623834306435643463333464623964 +37656465623039623434386538316366383862613939373938666466333863396336663765303034 +37323263613937313934653031663932623631656138653533613236653365383936613162383365 +66383262396638393062663563333630613239313766653265633234313830373039356231643961 +33643035313466613733333733653161333431376335666532323965613236333334303737396632 +64393136613966323232303030343739353665643036623734613066343131633131646633373161 +38303363646465313966323337383531356333393035373466646562346138653636373464643463 +63653962333333363339393334663031383262366531303761626439323164373763646539623966 +31323437363436396335386666333433333637656138353138313831643965646236666132653339 +32663765336333313439333538346236626466323761313962633638363937623163643933383833 +38653865303161653538306365356130636563393339643162663032623661323263313537326639 +33666534383435363636613066303563396662306236326531393930643736373038376562663139 +32346362363832343063373361346139666463636233316539326262316637336366373831376431 +62393630623466636564633866616231363030616238653130396163303832343632653436343235 +66656563643862633639313861636232376331623666623230303831633336373836343464636631 +30346161363338636261666337393739656238613964326365333934303030616534623539666537 +61333131373535333164303565386464313561373666316430393139303631376165666230653364 +30366138623532626530366166376663363435306465303064616230306234313961626630663034 +38613735353332356437333763616437306535666434356132646164323766303665613963653233 +34363136363733643134356635623437646230653737393966323464636530333036656134373431 +33653931653238333062336362363233336137333639653937653866663837316363383266643434 +30323533373930323735363561343437383839636561653832643732616636633765316136353738 +62363066336664633936343563393166653037336164373935656533623765623830386664623438 +39343664356635666436353136653138353330623533663032633562336538623632373639613165 +38643530623435346466373965636466373963623434396138366131363863346139656538353362 +30323235383565336538386666663461626435356564396130326662383764323766653761313164 +62663761613036623131636363353437643864343431656164303461653533646462643130353132 +35303164363736393434633635306534326237393136653163326164316431616661656535333263 +36343430356261616233613862303263303463353366666636353434656333613964373937356136 +66306364396233356138623566373034366236323863643139376234626432623965643361623539 +37383566643165333832626661303030383031613434353331303934616233383937666535633039 +62323333373866303634363863313762636165363762396562316538383137326263303665383230 +36393863633562393239633862633330666436333738646661613531373837656139386635356161 +38373335333833306538613132666261376366626132333935353931333563616164653664656466 +37333632376333666232633561353366353231383634653133616234333136323061333737353735 +64313735343865616436363436393231303937663135323238346139363337623766346365333737 +61656466333731663930656538633861376535303564653531313565663836666634613735633562 +61326237633761633062613731386364356439386664323564666263326262656434656136336166 +33363333646139346263666335343330383431653363313131343939303237326638613137386430 +34303561323766323534373636343435393863613465353061633661316231346332636364346166 +37326364396438316233346264623238663630303832653263333237333839613266643931353933 +30353339303666386263326264346531323465353331396438643837623633623466356162663439 +36633035636630313831363930306562383362353333646138333264376332373930396530613161 +33326163313838323966633735653939353031643761613238316535646233316137303161343763 +64383530356630343438363564326135633933643637613763346130326535306632636365623831 +31623931643430666537666339313032313164393930386134326530666361363264623766303236 +37643763363262656166333430663038326165333635643763636361376565363534346462613236 +36386263356133643635613931383939633030366233303934353532316163333966353239326563 +62343265313434663265633739316338373137363432313662366661326134303365393732373930 +66333337393565393166333036663863353764633436613637666130666562623666666338356532 +64663239306335373738366633396233313139633138316332363063323237386166353734313363 +66656161303066333238326232313534623661656530373238633632346636303764316333343963 +35386666366437363433613763666438623637663634393966626561653438663133326134623364 +38303332643930366663613433623161356364653434663163633938376632303861353066363562 +31366338656535346435623162626133343431623864663430383032343933363538383763336639 +36303238633362643331353261366437363239386538633430316162323063663436613134613937 +39326464346662356566653634353034653063623138336630616637376666323864626163383763 +62356439623233363465306432626465363466643863336433356138386266303562656530336562 +66613332643132363135313664393131303733626463616235363735343365316231373135326433 +39393562373935656133383266356538623464373565626136393834366233613062323736316465 +65656431633462626536353263353634383862633637323531396630633061373930393637366364 +62363132626266613662303462643436623366666161366536333838373131336336306131633131 +35623139373230303235303662663362616466326435336334363864323966386166633535623237 +38643936323438643731616338326636323138663362656666383539623330616537623663316438 +37336162666665343433303133356464653736646463316531633936616332663661393933356334 +66663965643562323830363236626266663939343635346661643130356165346565373231623734 +31383432666137326639653337336262326136613132306233323263626263623435643436343165 +36353765386138353136363031633066336337326563393938353534633339316663303938653432 +36353935333066376663643336653661313262613930626130323462633463323762303832333732 +36376138666632323762323334666438663665623231653732646335663832326138656664646133 +66396533633539623762353439333930636434306364323062333130366463336430316265376336 +36396631396534666633663532376364613161613366383766623830363361346666376238333539 +63386161386565313035643036656264636632306133643135633531616263656162323664656466 +61633235666535346663623739313262333762323936386537656235326439363835643363646166 +35666335616337323733326231656431373835386135353737343436383764356431623664383039 +63363235333337313861363632613131326432393035313539356132623666333562653361396332 +65313439326632666530336337366431633631336539366634616330303764326163663930373861 +34313062323730333437346630646331383330333036633532323333633361633362343137326134 +32663938346531306364616631343161656666343464336333356239333962666563373839333238 +37303339356130633630383961383236383166336265633736366432653835646364383062663463 +37396562386361396639613735303436336630633436633363336166326639326535623637346230 +32653233646132613436653065663633336131623632376632656562643461316539383532326234 +39616661316235663639333938306536366137373834313938363863663739363230383730343736 +64323533643630346463663439396262626536306130623938616164623466373232376362373132 +30353337313531313731656533356363366266383964386432336662323636356565643665663738 +64396338376137336139626664313366383335376335363234636562356332643463396333616333 +66356566386563343463313837313337383066653339363930356237383866376661376432333037 +39663935646137633962396235323238653830343337313966333763326563396234356139353365 +64336464623835316333623033393338656332356363363032613731373639383866323636356436 +33313430303865386630393166363530643862393031646166323238666537333965346661613565 +36396666653635376533616363383930333938663636653834656134623636643735646334643035 +34656433643464373538366238643564363564653638333632646139363564643064343134633738 +66633138333462346135393232663036323165303166393966666466616436366331666436656535 +61393763353366643438646134306363373834663638643963356132353237663834306564663332 +35663661313338303265396664393166363432653562323866636364356461386461636132653362 +34633563323566383666323739353230376231363734383161333132663635356339653662326438 +35333135633865633465383265343834316531303338323061373533323262353466326163366438 +30366239306538306564383465646163343538306336643733663362353337376538396362393733 +64343238316263663966643763316631663630386536633538336363633431393131383264666163 +38653236303035383033343331353336623131346534613465626566623638646334363333316161 +61376461666535623130666530393063633761306530303466326364336263313531386338626338 +31653266646235363562616130636566613630383130616431336639316433303166353338363433 +31626337343264663562666133653836633238633335373539396134636337343864323562303635 +38356338633035633037316565333635326530323661363565646362646263336137633736366437 +62623131646132363961633462313131346636313237363561623634333737343131303065313632 +62343937336266313733643137303061336433353363643138666434383935616336363336343035 +64616466633739363339393535303965666266313563303965353166623261306636363465366537 +62306634323238356332616437623735313264336361353738336530316633663665363439636333 +62626139356633316632356138653636643461376364616265383466613435313532366231653030 +61366161356238653762373665623638636339636530303239356232626362313866383838353134 +37336632396161366164393634636366393837373465666134623166323934346237613935636231 +36613235636564346663313363626564356435616333643865356232306338393962613732636537 +33653031343432326463393765636634636638363763303930323931303436323665366266613536 +39366632313930346465343130376533393166353436306662393433346439383935636338323432 +62356234303638346162646533613265306534326538653630303838633339326537333961636632 +33376561373332323763316637346665613237343965636430333262623164616164643963393236 +62323337616132363134393335663636333537333163653539336565653232366561386364323966 +32376664376532313865643038333765353639376265333866613866393737666434363638656438 +34306336616633623834316433333835393161616666316262343235356437343632353634646639 +34393966323365663034323832653737396632656234346135333730646333366536623064633533 +66376463363935366533613132653038363263633465353632613663643965396336386261386334 +62666330656238303832663631316130326635336437663233333136626139303231356233356530 +65346233613662656533623338363666306663303638356439636135663830666630336235623061 +31303565396232663438336166353665663530323338353062336563323735333133373530643964 +62666335643535353566343433633838306431303330333032646662623132366163343233326230 +65343637393764326361333336633538623263343539323962333464636664393031323930346462 +30353335313839353739346661636630313230333735653830326239663838633763346130356531 +31363338626537313735366662336438396237356434646635663364633161616362376333383562 +65323830613736303065623561383939333734343838666261323735336163313634366164353463 +63633439346532656161333761363764313138313138353630383565393734313732656331396365 +34373661323631356164333564383832316332373134643737346163613130363732646334376631 +64383964313337363337643330363132333032326566633836653337333435393930386431636264 +61393531663439373436383666363963643834346331366562343534363230366135393733666537 +37373538353664623265313432386265373662393737356331363838356234653838623735636639 +32623731336361333934613430616334623232343562353366393937373830666365616239353637 +62373539313838663333353766623932613237636331316437303431623630626131653633383261 +39643335316130306163343435383238616563353662366562373265323961373336653263346337 +62653865643930616634613938316530613138363834333261626539363032333935646438343265 +65613662643662346639323631333765626338636633326335313031663531373331386564383231 +35383761333633353439323937376531383865633533613233393966626463313537346566646434 +30643566396336356531303636623633393461376364663366326238626465643130613335323761 +30646565633438636235643635343164653831363238326435663133656238336564616666326163 +62353033643531316432663935666433346163323333396464383262383836303965343663356366 +61313639343766383730346535616132613165346333326237376631393065303266353535346635 +31613731303739613639396138343635306263313434353563386462313936396261633334313665 +63336435646264376666626438643864613162653764363038616662616438633231343837613937 +32613732303262316634353539366330663236633038343565623163623335323462363836616362 +66333637343266623365336531383664613962666236626565636138343063343166643435636331 +39353037393335636235323531346538386338303131316637346361316534333863356265623966 +62666538306261343735353130643338306232623030303864666539653739336166346133616664 +62383331313865633362376338336663303734366335316562633365373165356237366365626136 +34316138383731616163646432656338363165376134323163353862363162303066636536346137 +36356439623234663766376466323662343036636163613737346537633964353533636139633364 +62633064363565626262373364363132316636663334366236653632383265326135653732303464 +64323032396163616165353535363766646331393730643066383536626234363136653832663664 +33313865666665313334636530396634633435626265353037336334383239636363303561333136 +30336432643637656464393639386462616231366364353065626635373462303466666638353433 +31313736643931653663323130653337366530303032386535356563663731653366306331666537 +30323165643535643037616537373664626533326263393266393264313530376466323266313735 +33383761336162323563653833376333336530386266646432326336656637356638366138363361 +64626435393634373065656431316436666135383461326234343031326430373137383037353661 +61353962613131336535636465633064316534336531333238313536623064303838663339363636 +39646232653064633634316264653164643731323332313666333066303837316533623264303363 +37373837663836666436326236633338323134383636626539333935666537656261313664326633 +62323636333462303966363563373066656632633265636364323835343735336361343561613334 +33376534636236383264356663626264323138353635353663623135623562313764323331343265 +65353037313266353035313739373932353035626435373638343466373137393230303434303635 +35346530666437323936653435663736383664633230653633613264636562356662336236623137 +37343464323466316634353663333562386432653939323235333566363239383231363131383736 +32366436626163643065636363376338623930316134616534656232336235306532663061313238 +31306133326131313738353835383364326566376233313234326264346561396137636236373730 +65343433346262363931316236306530353137656135346339393335363732636534393235326535 +61356637323730373838646132373939666639663364346439383537303236363939633665366330 +61656432623537383835656234346162643166656235636234303738613166316535353163663438 +33333639623463376335663763343365616466326233643131643739646633373637343035363863 +62343030626131666139646237636335636161623332633032633635666432343432323365303139 +63646264643731356539316632366366303532396238353938313036616435383035306264366264 +36356333396437616462363930313239313961336439633666636337616237656133376631363336 +66383637303038646633346134313365306162656339653161303661393938613939313231376564 +66613362303062313530306237656532366434663134323635376436643738383366306663336335 +64643039346135646463323962646639303037363239616661666664356636383561623264353762 +39653231396434383866376434663961366665633766616261393634643435373762643630303066 +65656234646366643062356231666636373839306630653038653861386538313539393332383762 +39613130366439393064343136326362653962626262363132616231646635306635346532343838 +63666165666139633563306532303761306133636366376135643565303862393365626637343663 +37323437373962396366353464616432313533643731656665636637373138333936343238366130 +38316661653434333164393633303838393563363861663138373939623233303432353933323736 +64366364653266663433643062326331326331313565623561383865613764393066373235643963 +35616436643362613565336630393838363031663134313935353539643138396366333935613363 +31663464303234386435303236396237643465383530316435303932636463396364346535623063 +37646661633464346139303932353061356566343063326661636662343135333865303166386262 +65346536376135653662393732346535646261643161666136303961323834346565356336666438 +65386261366463653165376135363239323538316261663065396534356130356562383634663966 +35343530323039383335623034336161326162326135303535366536393063383439353436663664 +31393536616539363163623536376266653434376661656237356332346530653333363139663163 +36356364333065623764353233316363396231666530353165396532363934376165396633306230 +30653038353064636630373637393766313330343364656330313737663935366263333731393830 +30356635643931303666396131616363643734616231393136643463373734353239393934303066 +33663563373837633031663765663235363463363166306261646638623165383430626633393336 +38303038323937323261646432363564616537666431373336386236366461633733613435326535 +38316230656138303565643861303238333434313963366436313662623764343065383461373634 +66616231373533613333326661356361393331393065316533656337323637376633373538373936 +65333934326437356264636563356661376564303433363830343462396134393434353633613363 +38663232353439316531323136323864383362373730306635333062333239363431383930616566 +63363132613066363930613163613763636637633863376265383438393462363161313036623738 +63663034333931653432376233353330373564366230323830613062323866333763326431323466 +36333532333862616632386631356335653035336265653535653363643363333535656430366138 +64333665373330343665323837396633313533666163373764323632636631353334623530376638 +64643138396632356662653230616563623830323762393231343437663363313931326232333865 +65353363376436646161636561393364613961666337353035623230303636303831383234613862 +32633030343833313964616536303733396134376234636439653639376365386265636238653639 +63313362333538386265313833663639626134616438303435386265376237383036353564646262 +64633831346236393730353036393331613863363066333532613134646132643437393736623566 +37633233303335646131663463323837373465643863386361616264363566363337393862353030 +63366532383963623363623330303538613132386335656365343436643661623031646331373262 +38303463326363373563643232396266636365366330623564356562653032396564396266623837 +64306264643239326236633338623066656531396666656330316162396432643231623336366362 +61633961653065356231343636666432316163386131363364333538333366376132393064636663 +66646363623335366631636234366136303764393235643934336137646666623236353366393532 +61616362326433333565626363346462653532616137326532643337326437353965316566633961 +38333634666232386332653032366337383530396533656238653062336331333762663238383739 +31313338303562373036333336663831346330383361306330386438363832626437383064326537 +32313638333933333734636437613764316437373961626335373731386433323138336362313562 +38613965383738393130363035626436303130326363613566653361363333636263633266366433 +33643530663266356562316630666536303834393065633930663535363364393132646664363839 +31353136363838326661343337633731333737366665626533623336616462343063393330353566 +30383032643662333630306238396665363861333336633936663034373831393565376564323331 +38666461646633666334333739316265633134316332643930626264303938653535613131326234 +31636463663930643539346362663166656361316537623565303037613738366237653432306362 +36346461383732656333633132373663636231373730336462663438313835336335376164623238 +38326637653536313262356234663566613064643135663533376336353231646137363332346338 +36396433303839623833363238343433343866376433303939343563336562633631633331323666 +64306335313137306639333231383232333435343335663862646466363432613232363939633335 +35643939366137303234373961363539626265396431343562353163323439303535613662646531 +65613362333962313735373839316636313638303563316236326362326266383738633462373763 +33373731303335386439306462656363623138333530643638356530623936386261363731663936 +64663235326438333135353563373961396566303764313239623331636261343463366431613437 +66383563623634383836316366633739323938353739313236326361333433616163343563353136 +35313939666234643430336237383932383833343061663565633066356138666431386434323339 +66316537653362353239376433653230316261613633633564373163386338336531303236653139 +33346536323561656464323831303062636337666233623737386661336133353932666666656364 +61333032623133383661333837353233343537313739633963653837306662303761363538646534 +65353631366364666363353964643531623061383462646634666633366665303236386666633634 +32633164353830366530333733623934633764363731373831623835613464363966646531333166 +32303165306337363938393532393730633165333438646366333836623965366666396330663363 +65323930343936343563633066343766643165316233323934363735363538313862643938633839 +61373735373535353237626238346135326336393666316639623961386264323735666666613535 +37383738353630373665636137316165323436396637643138623935373864633161353733396339 +30313366663465646131363737633433393064626266383765613431623639336233623765663239 +31656166353230326331306362343733376633656430373537346134373935663062663132343231 +65626133356365663830626333366339653934636663343136336237626535333034623135643530 +65653764383935653735326362343637613462623539653737613935313534643037306364663865 +66323837393135373261663564376530633739643035653862383639303339383233326636363366 +65613830613063376634636334343130373264346137386165303934646266343066383633383435 +37396134626164626539396232313564316635653237316561653730353732393765633165643734 +31326534656339393238356465386135656232626630656236383138623964323530653532616239 +31373061623539336636643135636538663732303930346230643365313938303831666234376162 +63636362303332326536653862356237313438616161356362323662373464643635313261626634 +33326661386638613835646666363663356665643839376365336335373432623033356334376530 +66663064333230306637633261316566633163313261663136363261373332316635366564393734 +39386361613135386336376162313035346637333464333431366133383166363063643237323366 +65343437393561646438363163653532313031373536613734353830383835393032383532396264 +31383634663038363139633333396436323363656666386535653035386237396535306434663364 +64363438616166643464353765356265616534333830393630666164356565646265323632623532 +36346536386638653665373634326161386164376338663834373930333030376536363563373763 +63383266313833616631623363663130376165386462623161366533376466633536366561643130 +66333134633164613266373965313261306365663762386563373230633762393662323332653937 +66373230373830313262393030303537313262333233633033306335353961383231326666636263 +62313332303733633038653733323933316161356531353733356431333034383037386366353038 +33386130316633643034666364316366346131616636306330373763623130393136623066346365 +32393330653266636665616131613666376635353137303766313132313530326439656363366639 +64653535346461636565333161663062623831383662303165353236653239623837386130616464 +38376334326330396638333764333433376232326361613631363635366236633964396430343330 +39336232303961653364393939653937643936336331646239666232653961323038643961383939 +36303937393362393539313034646461656464373266636462646162633632353736653631393434 +6631 diff --git a/dev-tools/ansible/inventories/staging/group_vars/keycloak/vars.yml b/dev-tools/ansible/inventories/staging/group_vars/keycloak/vars.yml new file mode 100644 index 0000000000..2ee750c5aa --- /dev/null +++ b/dev-tools/ansible/inventories/staging/group_vars/keycloak/vars.yml @@ -0,0 +1,12 @@ +--- +# Keycloak server configuration for staging environment + +# Keycloak database connection (points to database server) +keycloak_db_host: "xxx.xxx.xxx.xx" +keycloak_db_port: "3306" +keycloak_db_schema_name: "keycloak" + +# Keycloak virtual host hostname (for Let's Encrypt certificate) +keycloak_vhost_servername: "auth.staging.cybershuttle.org" + + diff --git a/dev-tools/ansible/inventories/staging/host_vars/api-server/vault.yml b/dev-tools/ansible/inventories/staging/host_vars/api-server/vault.yml new file mode 100644 index 0000000000..a5e6fa9971 --- /dev/null +++ b/dev-tools/ansible/inventories/staging/host_vars/api-server/vault.yml @@ -0,0 +1,20 @@ +$ANSIBLE_VAULT;1.1;AES256 +30613833613339306564376233636363653334363264316639303234316163333632363033353065 +3962376364316136343430646437616134653437386238620a356563326231623631373566346434 +37343366663630346464356332633135363635303236333336623634633865383538346538626639 +3863333836666530340a636363323061393462643238633033373730373335313035656535313334 +36386532616439363439343538356662323766363333346165356334633635303637626539323532 +62326666343632383531663365343664313930343139326137373565303437373630383132653334 +32336230383361336262663839376135393235383933356561323530396164343331616136616163 +65313265613730626133353930363234616130393035666330633261346638633733653037383438 +61353633643530626362323639363835316265303730363364323363363438346163643637643563 +31326232626336653466393064316231366466613465663063643139636666303266316536333634 +64646362633733386332303433316631313936386235346634623062313338633537666432653865 +37613339396266626333643836343030326562323033363234656366343063303130386234623662 +36396239656539303365636539316530626563396665666134616430336430386330633366303864 +36373366383033303332363263343831326463383238633766323862393336326665643162343062 +61376139346538383965336334326563323332323936353439663935323238326436353162376336 +36323030333262633764633862613765643934303430613438326535666235343535663735373639 +62333864356630376537306163623536613330623537613239663133643065323533653561643435 +33636431373962363634336536363437656433306564653836663431323331386137646630316439 +346666336436373432633264303934326139 diff --git a/dev-tools/ansible/inventories/staging/host_vars/db-server/vault.yml b/dev-tools/ansible/inventories/staging/host_vars/db-server/vault.yml new file mode 100644 index 0000000000..c9df27192a --- /dev/null +++ b/dev-tools/ansible/inventories/staging/host_vars/db-server/vault.yml @@ -0,0 +1,19 @@ +$ANSIBLE_VAULT;1.1;AES256 +33383332313133633831393362623362383462333965333337316334646435626536663865636437 +6539623961356236343037383364333632383037383638390a366236346166353338663036636462 +37303064656639383333313862353835323832663561316537363731303736323430333933326233 +3434376538346438620a383930363462373263333235393532313161643562636566626136373366 +31306532326463646433663938373830346661393734323164396637343263353231623265663031 +61313031343166373932386462363431666632623163636162633462323130326265313330653363 +64626539663164353533336338393330653139653665366639346338343439376232386538366439 +61356431303664653439306338636561643461636230636662303637353233356235313134343639 +36376536613261383962633635363436366138623938363930343063343932313730373231306163 +31386633346166626261613764313334666433396336366131626631623630623863393434356337 +64336231306362626433343530383264636433613633323230373865623433363134383535393365 +31313531373231356561383265356539373530653632323139616334353766376637326164613466 +62636362323636626335306161363730316631666537316435353365353839346533303638613465 +64333835643832333562636133386236613861313264396239656666373035623839396463346266 +65636430616330643636613739353439643334396230623431663134333535323738303763653766 +36343164646438383061393833396639633931666563643063323835303761633865303562363665 +38373432633130316536303166393137313139353566613665633234393463663939646630343362 +3462653936393465363663363363383139643061623266363537 diff --git a/dev-tools/ansible/inventories/staging/host_vars/keycloak-server/vault.yml b/dev-tools/ansible/inventories/staging/host_vars/keycloak-server/vault.yml new file mode 100644 index 0000000000..26bae194ba --- /dev/null +++ b/dev-tools/ansible/inventories/staging/host_vars/keycloak-server/vault.yml @@ -0,0 +1,19 @@ +$ANSIBLE_VAULT;1.1;AES256 +61376631616161353462343462316538376632616463376630343332373432383835313363653565 +6136326631383532653163626337616661326432356364630a633636613835616136333939386463 +33373035383563643536666262663661336637393231663231623534393837303634636135613732 +3466653134363039640a326232393938633134383538386131306364626561383033313761336634 +39326337626463396533613332303331343736333735636462666363626630316362323735376163 +32653862353031333839376235346131333539613830313933333738616636343731396666656333 +66396165643633353534373563313861386639313363343165366432623535303737326464666665 +37613862636332393437383533373334646366663539373931613364353935626663343639666363 +65613639363639346463333031313764653934353565323662303038363039333638346363336365 +31633231393635396635666461636338323138363834343932353930626531346337323435383462 +66653233326336363133636530303330323464616138613430353966353662663366323462653162 +30636533373639366234333234623635323736623934666166373637366334353731376339393634 +63323863393862643131656264343238343531633337626234366235386331383631363565363662 +61666631626162636561373332353736356639326566313561623732656631316338376266623563 +31306234353566366331613565383136623163393230373239383630656635356530326233646337 +31663436626337353632353332663061383232646530366362383034653638316130396339666533 +66643066396566626237356431623163623538356632333733346334353735643766353764303433 +3063663237646466353765663438363237333630383763303534 diff --git a/dev-tools/ansible/inventories/staging/hosts b/dev-tools/ansible/inventories/staging/hosts new file mode 100644 index 0000000000..6be9cfb825 --- /dev/null +++ b/dev-tools/ansible/inventories/staging/hosts @@ -0,0 +1,17 @@ +[database] +db-server + +[keycloak] +keycloak-server + +[airavata_servers] +api-server + +[zookeeper] +api-server + +[rabbitmq] +api-server + +[api-orch] +api-server diff --git a/dev-tools/ansible/inventories/template/README.md b/dev-tools/ansible/inventories/template/README.md new file mode 100644 index 0000000000..0257532867 --- /dev/null +++ b/dev-tools/ansible/inventories/template/README.md @@ -0,0 +1,138 @@ +# Airavata Deployment Inventory Template + +This directory contains a template for creating new Airavata deployment inventories. + +## Quick Start for New Environment + +### 1. Copy this template + +```bash +cp -r inventories/template inventories/my-env +cd inventories/my-env +``` + +### 2. Rename example files + +```bash +mv hosts.example hosts +mv group_vars/all/vars.yml.example group_vars/all/vars.yml +mv group_vars/all/vault.yml.example group_vars/all/vault.yml +mv host_vars/airavata-server/vault.yml.example host_vars/airavata-server/vault.yml +``` + +### 3. Edit configuration files + +Edit all files and replace `CHANGEME` values with your actual values: + +- **hosts** - Replace `airavata-server` with your host alias if needed +- **group_vars/all/vars.yml** - Set non-sensitive configuration values +- **group_vars/all/vault.yml** - Set sensitive values (passwords, URLs, etc.) +- **host_vars/airavata-server/vault.yml** - Set server IP addresses and SSH credentials + +### 4. Encrypt sensitive files + +Encrypt the vault files to protect sensitive information: + +```bash +cd ../my-env + +# Encrypt group variables (database passwords, API keys, etc.) +ansible-vault encrypt group_vars/all/vault.yml + +# Encrypt host variables (server IPs, SSH keys) +ansible-vault encrypt host_vars/airavata-server/vault.yml +``` + +### 5. Test connection + +Verify you can connect to your server: + +```bash +ansible-playbook -i inventories/my-env --list-hosts -m ping --ask-vault-pass +``` + +### 6. Deploy + +**For initial setup (full environment from scratch):** +```bash +cd ../.. +ansible-playbook -i inventories/my-env airavata_setup.yml --ask-vault-pass +``` + +**For service updates (infrastructure already exists):** +```bash +cd ../.. +ansible-playbook -i inventories/my-env airavata_update.yml --ask-vault-pass +``` + +## File Structure + +``` +my-env/ +├── hosts # Host definitions +├── group_vars/ +│ └── all/ +│ ├── vars.yml # Non-sensitive variables +│ └── vault.yml # Encrypted sensitive variables +└── host_vars/ + └── airavata-server/ + └── vault.yml # Encrypted server-specific variables +``` + +## Key Configuration Points + +### Server Access (host_vars/airavata-server/vault.yml) +- `ansible_host` - Server IP address or hostname +- `ansible_user` - SSH user for deployment +- `ansible_ssh_private_key_file` - Path to SSH private key + +### Database Configuration (group_vars/all/vault.yml) +- All database passwords +- Database URLs and connection strings +- Server IP addresses embedded in URLs + +### Service Configuration (group_vars/all/vault.yml) +- IAM/Keycloak credentials +- OAuth client secrets +- RabbitMQ connection strings +- Email monitoring credentials +- Tunnel server tokens +- Keystore passwords + +### Non-Sensitive Configuration (group_vars/all/vars.yml) +- Service ports +- Build settings (git repository, branch, version) +- Paths and directories + +## Managing Vault Files + +**View an encrypted file:** +```bash +ansible-vault view group_vars/all/vault.yml +``` + +**Edit an encrypted file:** +```bash +ansible-vault edit group_vars/all/vault.yml +``` + +**Change vault password:** +```bash +ansible-vault rekey group_vars/all/vault.yml +``` + +## Troubleshooting + +**Issue: Playbook asks for vault password repeatedly** +- Check that all vault files are encrypted +- Verify the inventory directory path is correct + +**Issue: Connection refused** +- Verify `ansible_host` in host_vars is correct +- Check SSH key file path and permissions +- Ensure target server is accessible from your machine + +**Issue: Services don't start** +- Check logs in `deployment_dir/logs/` +- Verify all required ports are open +- Ensure database connectivity \ No newline at end of file diff --git a/dev-tools/ansible/inventories/template/group_vars/all/vars.yml.example b/dev-tools/ansible/inventories/template/group_vars/all/vars.yml.example new file mode 100644 index 0000000000..d9022a9a6a --- /dev/null +++ b/dev-tools/ansible/inventories/template/group_vars/all/vars.yml.example @@ -0,0 +1,28 @@ +--- +# Non-sensitive configuration variables +# Copy this file to vars.yml and update values as needed + +# Airavata version and build settings +airavata_version: "0.21-SNAPSHOT" +git_branch: "master" +airavata_git_repo: "https://github.com/apache/airavata.git" +airavata_source_dir: "/home/{{ deploy_user }}/airavata-src" +deployment_dir: "/home/{{ deploy_user }}/airavata" + +# Maven version +apache_maven_version: "apache-maven-3.9.11" + +# Non-sensitive service settings +deploy_user: "CHANGEME_DEPLOY_USER" +api_server_port: 8930 +profile_service_port: 8962 +registry_port: 8970 + +# Database info +registry_jdbc_driver: "org.mariadb.jdbc.Driver" +registry_jdbc_user: "root" +appcatalog_jdbc_user: "root" + +# Paths +local_data_location: "/home/{{ deploy_user }}/temp-storage" + diff --git a/dev-tools/ansible/inventories/template/group_vars/all/vault.yml.example b/dev-tools/ansible/inventories/template/group_vars/all/vault.yml.example new file mode 100644 index 0000000000..7faff4cff5 --- /dev/null +++ b/dev-tools/ansible/inventories/template/group_vars/all/vault.yml.example @@ -0,0 +1,115 @@ +--- +# Sensitive configuration variables (will be encrypted) +# Copy this file to vault.yml, fill in CHANGEME values, then encrypt: +# ansible-vault encrypt vault.yml + +# Database passwords +registry_jdbc_password: "CHANGEME_DB_PASSWORD" +appcatalog_jdbc_password: "CHANGEME_DB_PASSWORD" +replicacatalog_jdbc_password: "CHANGEME_DB_PASSWORD" +workflowcatalog_jdbc_password: "CHANGEME_DB_PASSWORD" +sharingcatalog_jdbc_password: "CHANGEME_DB_PASSWORD" +profile_service_jdbc_password: "CHANGEME_DB_PASSWORD" +credential_store_jdbc_password: "CHANGEME_DB_PASSWORD" + +# Database URLs +registry_jdbc_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/experiment_catalog" +appcatalog_jdbc_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/app_catalog" +replicacatalog_jdbc_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/replica_catalog" +workflowcatalog_jdbc_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/workflow_catalog" +sharingcatalog_jdbc_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/sharing_catalog" +profile_service_jdbc_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/profile_service" +credential_store_jdbc_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/credential_store" + +# IAM/Keycloak credentials +iam_server_url: "https://CHANGEME_IAM_SERVER" +iam_admin_username: "admin" +iam_admin_password: "CHANGEME_IAM_PASSWORD" + +# Keycloak admin account (for Keycloak 24+) +keycloak_master_account_username: "admin" +keycloak_master_account_password: "CHANGEME_KEYCLOAK_ADMIN_PASSWORD" + +# Keycloak database credentials +keycloak_db_username: "keycloak" +keycloak_db_password: "CHANGEME_KEYCLOAK_DB_PASSWORD" + +# Keycloak realm client configuration +# PGA client (for Airavata Gateway) +keycloak_pga_client_secret: "CHANGEME_PGA_CLIENT_SECRET" +keycloak_pga_redirect_uris: + - "https://CHANGEME_GATEWAY_HOST/callback-url" + - "https://CHANGEME_GATEWAY_HOST/" + - "https://CHANGEME_GATEWAY_HOST/auth/callback*" +keycloak_pga_web_origins: + - "https://CHANGEME_GATEWAY_HOST" + +# CS-JupyterLab client +keycloak_jupyterlab_client_secret: "CHANGEME_JUPYTERLAB_CLIENT_SECRET" +keycloak_jupyterlab_redirect_uris: + - "https://CHANGEME_JUPYTERLAB_HOST/hub/oauth_callback" + +# CILogon identity provider +keycloak_cilogon_client_id: "CHANGEME_CILOGON_CLIENT_ID" +keycloak_cilogon_client_secret: "CHANGEME_CILOGON_CLIENT_SECRET" + +# OAuth secrets +default_registry_password: "CHANGEME_REGISTRY_PASSWORD" +default_registry_oauth_client_secret: "CHANGEME_OAUTH_SECRET" + +# RabbitMQ +rabbitmq_user: "guest" +rabbitmq_password: "guest" +rabbitmq_vhost: "CHANGEME_VHOST" +rabbitmq_broker_url: "amqp://guest:guest@localhost:5672/CHANGEME_VHOST" + +# Zookeeper +zookeeper_connection: "localhost:2181" + +# Email monitoring +email_based_monitor_address: "CHANGEME_EMAIL_ADDRESS" +email_based_monitor_password: "CHANGEME_EMAIL_PASSWORD" + +# Kafka +kafka_broker_url: "localhost:9092" +job_status_publish_endpoint: "http://CHANGEME_API_HOST:8082/topics/helix-airavata-mq" + +# Tunnel tokens (for agent service) +tunnel_server_host: "CHANGEME_TUNNEL_HOST" +tunnel_server_port: 17000 +tunnel_server_token: "CHANGEME_TUNNEL_TOKEN" +tunnel_server_api_url: "http://CHANGEME_TUNNEL_HOST:8000" + +# Keystore passwords +keystore_password: "CHANGEME_KEYSTORE_PASSWORD" +credential_store_keystore_password: "CHANGEME_CRED_STORE_PASSWORD" + +# Agent service datasource +agent_service_datasource_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/app_catalog" +agent_service_datasource_username: "root" +agent_service_datasource_password: "CHANGEME_DB_PASSWORD" + +# Research service datasource +research_service_datasource_url: "jdbc:mariadb://CHANGEME_DB_HOST:3306/research_catalog" +research_service_datasource_username: "root" +research_service_datasource_password: "CHANGEME_DB_PASSWORD" + +# Agent service Airavata settings +agent_service_airavata_url: "localhost" +agent_service_airavata_port: 8930 +agent_service_airavata_secure: false + +# Research service URLs +research_hub_url: "https://CHANGEME_RESEARCH_HUB" +research_hub_dev_user: "airavata@apache.org" +research_hub_admin_api_key: "CHANGEME_API_KEY" +research_hub_limit: 10 +research_portal_dev_url: "https://CHANGEME_PORTAL" +research_portal_url: "https://CHANGEME_PORTAL" +openid_url: "https://CHANGEME_AUTH_SERVER/realms/default/.well-known/openid-configuration" +user_profile_server_url: "CHANGEME_PROFILE_SERVER" +user_profile_server_port: 8962 + +# Keystore file +vault_keystore_file: "files/airavata.sym.p12" + diff --git a/dev-tools/ansible/inventories/template/host_vars/airavata-server/vault.yml.example b/dev-tools/ansible/inventories/template/host_vars/airavata-server/vault.yml.example new file mode 100644 index 0000000000..477d50a07e --- /dev/null +++ b/dev-tools/ansible/inventories/template/host_vars/airavata-server/vault.yml.example @@ -0,0 +1,19 @@ +--- +# Server-specific sensitive variables (will be encrypted) +# Copy this file to vault.yml, fill in values, then encrypt: +# ansible-vault encrypt vault.yml + +# SSH connection details +ansible_host: "CHANGEME_SERVER_IP" +ansible_user: "CHANGEME_SSH_USER" +ansible_ssh_private_key_file: "CHANGEME_PATH_TO_SSH_KEY" + +# Optional: If using password-based SSH +# ansible_password: "CHANGEME_SSH_PASSWORD" + +# Optional: If SSH runs on non-standard port +# ansible_port: 22 + +# Optional: SSH connection timeout +# ansible_ssh_timeout: 30 + diff --git a/dev-tools/ansible/inventories/template/hosts.example b/dev-tools/ansible/inventories/template/hosts.example new file mode 100644 index 0000000000..9c1852ea78 --- /dev/null +++ b/dev-tools/ansible/inventories/template/hosts.example @@ -0,0 +1,12 @@ +[airavata_servers] +airavata-server + +[zookeeper] +airavata-server + +[rabbitmq] +airavata-server + +[database] +airavata-server + diff --git a/dev-tools/ansible/roles/airavata_services/defaults/main.yml b/dev-tools/ansible/roles/airavata_services/defaults/main.yml new file mode 100644 index 0000000000..00e6c58a0e --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/defaults/main.yml @@ -0,0 +1,220 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +# Airavata Services Role Default Variables + +# Service distribution names +airavata_version: "0.21-SNAPSHOT" +api_server_dist_name: "apache-airavata-api-server-{{ airavata_version }}" +agent_service_dist_name: "apache-airavata-agent-service-{{ airavata_version }}" +research_service_dist_name: "apache-airavata-research-service-{{ airavata_version }}" +file_server_dist_name: "apache-airavata-file-server-{{ airavata_version }}" +restproxy_dist_name: "apache-airavata-restproxy-{{ airavata_version }}" + +# Service directories +api_server_dir: "{{ deployment_dir }}/{{ api_server_dist_name }}" +agent_service_dir: "{{ deployment_dir }}/{{ agent_service_dist_name }}" +research_service_dir: "{{ deployment_dir }}/{{ research_service_dist_name }}" +file_server_dir: "{{ deployment_dir }}/{{ file_server_dist_name }}" +restproxy_dir: "{{ deployment_dir }}/{{ restproxy_dist_name }}" + +# Service ports +api_server_port: 8930 +profile_service_port: 8962 +registry_port: 8970 +registry_server_port: 8970 +sharing_registry_port: 7878 +cred_store_port: 8960 +agent_service_port: 18880 +agent_service_server_port: 18880 +research_service_port: 18899 +research_service_server_port: 18899 +file_server_port: 8050 +restproxy_port: 8082 + +# Service hosts +api_server_host: "0.0.0.0" +profile_service_host: "0.0.0.0" +agent_service_server_address: "0.0.0.0" +research_service_server_address: "0.0.0.0" + +# Monitoring ports +api_server_monitoring_port: 9097 +participant_monitoring_port: 9096 +pre_wm_monitoring_port: 9093 +post_wm_monitoring_port: 9094 + +# Monitoring hosts +api_server_monitoring_host: "localhost" +participant_monitoring_host: "localhost" +pre_workflow_manager_monitoring_host: "localhost" +post_workflow_manager_monitoring_host: "localhost" + +# Server hosts +orchestrator_server_host: "localhost" +regserver_server_host: "localhost" +sharing_registry_server_host: "localhost" +cred_store_server_host: "localhost" + +# Orchestrator configuration +orchestrator_class: "org.apache.airavata.orchestrator.server.OrchestratorServer" +orchestrator_server_port: 8940 +orchestrator_server_min_threads: 50 +job_validators: "org.apache.airavata.orchestrator.core.validator.impl.BatchQueueValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator" +enable_validation: true +host_scheduler: "org.apache.airavata.orchestrator.core.schedule.DefaultHostScheduler" + +# Registry server configuration +regserver_class: "org.apache.airavata.registry.api.service.RegistryAPIServer" + +# Sharing registry configuration +sharing_server_class: "org.apache.airavata.sharing.registry.server.SharingRegistryServer" +enable_sharing: true + +# Default registry user +default_registry_user: "default-admin" +default_registry_gateway: "default" +default_registry_oauth_client_id: "pga" +super_tenant_gatewayId: "default" + +# JDBC driver +registry_jdbc_driver: "org.mariadb.jdbc.Driver" +appcatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +replicacatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +workflowcatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +sharingcatalog_jdbc_driver: "org.mariadb.jdbc.Driver" +profile_service_jdbc_driver: "org.mariadb.jdbc.Driver" +credential_store_jdbc_driver: "org.mariadb.jdbc.Driver" + +# Security configuration +security_manager_class: "org.apache.airavata.service.security.KeyCloakSecurityManager" +TLS_enabled: false +TLS_client_timeout: 10000 +keystore_path: "keystores/airavata.p12" +authz_cache_enabled: true +authz_cache_manager_class: "org.apache.airavata.service.security.authzcache.DefaultAuthzCacheManager" +in_memory_cache_size: 1000 + +# Keystore configuration +credential_store_keystore_url: "keystores/airavata.sym.p12" +credential_store_keystore_alias: "airavata" + +# Job notification configuration +job_notification_enable: true +job_notification_emailids: "" +prefetch_count: 200 +durable_queue: false + +# RabbitMQ configuration +rabbitmq_server: "localhost" +rabbitmq_status_exchange_name: "status_exchange" +rabbitmq_process_exchange_name: "process_exchange" +rabbitmq_experiment_exchange_name: "experiment_exchange" +experiment_launch_queue: "experiment_launch" + +# Zookeeper configuration +embedded_zk: false + +# Helix configuration +helix_cluster_name: "AiravataCluster" +helix_controller_name: "AiravataController" +helix_participant_name: "AiravataParticipant" +participant_monitoring_enabled: true + +# Job monitor configuration +enable_realtime_monitor: true +realtime_monitor_broker_consumer_group: "monitor" +realtime_monitor_broker_topic: "helix-airavata-mq" +job_monitor_broker_consumer_group: "MonitoringConsumer" +job_monitor_broker_topic: "monitoring-data" +job_monitor_broker_publisher_id: "AiravataMonitorPublisher" +job_monitor_email_publisher_id: "EmailBasedProducer" +job_monitor_realtime_publisher_id: "RealtimeProducer" + +# Email monitor configuration +email_based_monitor_host: "imap.gmail.com" +email_based_monitor_store_protocol: "imaps" +email_based_monitor_folder_name: "INBOX" +email_expiration_minutes: 60 +email_based_monitoring_period: 10000 + +# Log management +archive_logs_on_stop: true + +# Pre-workflow manager configuration +pre_workflow_manager_loadbalance_clusters: false +pre_workflow_manager_monitoring_enabled: true +pre_workflow_manager_name: "AiravataPreWM" + +# Post-workflow manager configuration +post_workflow_manager_loadbalance_clusters: false +post_workflow_manager_monitoring_enabled: true +post_workflow_manager_name: "AiravataPostWM" + +# Parser-workflow configuration +data_parser_delete_container: true +data_parser_broker_consumer_group: "CHANGE_ME" +data_parser_topic: "CHANGE_ME" +data_parser_storage_resource_id: "CHANGE_ME" + +# Monitoring and scanning configuration +cluster_status_monitoring_enable: false +metaschedluer_job_scanning_enable: false +data_analyzer_job_scanning_enable: false + +# Data staging configuration +enable_streaming_transfer: false + +# Thrift client pool configuration +thrift_client_pool_abandoned_removal_enabled: true +thrift_client_pool_abandoned_removal_logged: false + +# DB Event Manager +db_event_manager_class: "org.apache.airavata.db.event.manager.DBEventManagerRunner" + +# Agent service configuration +agent_service_grpc_host: "localhost" # Override in inventory files (vars.yml or vault.yml) per environment +agent_service_grpc_port: 19900 +agent_service_grpc_max_inbound_message_size: 10485760 +agent_service_max_file_size: "200MB" +agent_service_max_request_size: "200MB" +agent_service_pool_name: "AppCatalogPool" +agent_service_leak_detection_threshold: 20000 +agent_service_ddl_auto: "create" + +# Research service configuration +research_service_grpc_port: 19908 +research_service_max_file_size: "200MB" +research_service_max_request_size: "200MB" +research_service_pool_name: "ResearchCatalogPool" +research_service_leak_detection_threshold: 20000 +research_service_ddl_auto: "validate" + +# File server configuration +file_server_max_file_size: "12GB" +file_server_max_request_size: "12GB" +file_server_file_size_threshold: "2MB" + +# Storage configuration +local_data_location: "/home/{{ deploy_user }}/temp-storage" +agent_service_storage_resource_id: "default_9c15d8af-3d36-4c3c-a07a-0f3b4bb5b903" +agent_service_storage_path: "/var/www/portals/gateway-user-data" +agent_service_application_interface_id: "AiravataAgent_3eeb580b-b0c6-4f7e-8e3d-22c4f84ec3f1" diff --git a/dev-tools/ansible/roles/airavata_services/handlers/main.yml b/dev-tools/ansible/roles/airavata_services/handlers/main.yml new file mode 100644 index 0000000000..087ccde391 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/handlers/main.yml @@ -0,0 +1,23 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +# Handlers for Airavata Services role + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/build.yml b/dev-tools/ansible/roles/airavata_services/tasks/build.yml new file mode 100644 index 0000000000..f5f9d410c1 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/build.yml @@ -0,0 +1,69 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Pull latest code from git + git: + repo: "{{ airavata_git_repo }}" + dest: "{{ airavata_source_dir }}" + version: "{{ git_branch }}" + update: yes + register: git_pull_result + tags: + - build + +- name: Display git pull result + debug: + msg: "Git pull completed. Changed: {{ git_pull_result.changed }}" + +- name: Build Airavata with Maven + shell: | + source /etc/profile.d/maven.sh 2>/dev/null || true + /opt/{{ apache_maven_version }}/bin/mvn clean install -DskipTests + args: + chdir: "{{ airavata_source_dir }}" + environment: + MAVEN_OPTS: "-Xmx2048m" + PATH: "/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/{{ apache_maven_version }}/bin:{{ ansible_env.PATH }}" + register: maven_build_result + tags: + - build + +- name: Display maven build result + debug: + msg: "Maven build completed successfully" + +- name: Check for built distributions + find: + paths: "{{ airavata_source_dir }}/distribution" + patterns: "apache-airavata-*.tar.gz" + register: distribution_files + tags: + - build + +- name: Verify required distribution files exist + assert: + that: + - distribution_files.matched >= 5 + fail_msg: "Expected at least 5 distribution files, found {{ distribution_files.matched }}" + success_msg: "Found {{ distribution_files.matched }} distribution files" + tags: + - build + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/deploy_agent_service.yml b/dev-tools/ansible/roles/airavata_services/tasks/deploy_agent_service.yml new file mode 100644 index 0000000000..6e603411f9 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/deploy_agent_service.yml @@ -0,0 +1,78 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Remove old Agent Service deployment if exists + file: + path: "{{ agent_service_dir }}" + state: absent + tags: + - deploy + - agent-service + +- name: Extract Agent Service distribution + unarchive: + src: "{{ airavata_source_dir }}/distribution/apache-airavata-agent-service-{{ airavata_version }}.tar.gz" + dest: "{{ deployment_dir }}" + remote_src: yes + tags: + - deploy + - agent-service + +- name: Create configuration directory for Agent Service + file: + path: "{{ agent_service_dir }}/conf" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + tags: + - deploy + - agent-service + +- name: Deploy application.yml for Agent Service + template: + src: application-agent-service.yml.j2 + dest: "{{ agent_service_dir }}/conf/application.yml" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - agent-service + +- name: Deploy log4j2.xml for Agent Service + template: + src: log4j2.xml.j2 + dest: "{{ agent_service_dir }}/conf/log4j2.xml" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - agent-service + +- name: Set executable permissions on Agent Service script + file: + path: "{{ agent_service_dir }}/bin/agent-service.sh" + mode: "0755" + tags: + - deploy + - agent-service + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/deploy_api_server.yml b/dev-tools/ansible/roles/airavata_services/tasks/deploy_api_server.yml new file mode 100644 index 0000000000..9307f09550 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/deploy_api_server.yml @@ -0,0 +1,136 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Remove old API Server deployment if exists + file: + path: "{{ api_server_dir }}" + state: absent + tags: + - deploy + - api-server + +- name: Extract API Server distribution + unarchive: + src: "{{ airavata_source_dir }}/distribution/apache-airavata-api-server-{{ airavata_version }}.tar.gz" + dest: "{{ deployment_dir }}" + remote_src: yes + tags: + - deploy + - api-server + +- name: Create configuration directory for API Server + file: + path: "{{ api_server_dir }}/conf" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + tags: + - deploy + - api-server + +- name: Create keystores directory for API Server + file: + path: "{{ api_server_dir }}/conf/keystores" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + tags: + - deploy + - api-server + +- name: Deploy airavata-server.properties + template: + src: airavata-server.properties.j2 + dest: "{{ api_server_dir }}/conf/airavata-server.properties" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - api-server + +- name: Deploy email-config.yml + template: + src: email-config.yml.j2 + dest: "{{ api_server_dir }}/conf/email-config.yml" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - api-server + +- name: Deploy log4j2.xml + template: + src: log4j2.xml.j2 + dest: "{{ api_server_dir }}/conf/log4j2.xml" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - api-server + +# If keystore already exists in inventory files, copy it otherwise generate it +- name: Check if keystore exists in inventory files + stat: + path: "{{ inventory_dir }}/files/{{ vault_keystore_file }}" + register: keystore_file_exists + tags: + - deploy + - api-server + +- name: Copy keystore file from inventory (if provided) + copy: + src: "{{ inventory_dir }}/files/{{ vault_keystore_file }}" + dest: "{{ api_server_dir }}/conf/keystores/{{ vault_keystore_file | basename }}" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0600" + decrypt: yes + when: vault_keystore_file is defined and keystore_file_exists.stat.exists + tags: + - deploy + - api-server + +- name: Generate keystore from Let's Encrypt certificates + include_tasks: generate_keystore.yml + when: vault_keystore_file is defined and not keystore_file_exists.stat.exists + tags: + - deploy + - api-server + +- name: Set executable permissions on API Server scripts + file: + path: "{{ api_server_dir }}/bin/{{ item }}" + mode: "0755" + loop: + - orchestrator.sh + - controller.sh + - participant.sh + - pre-wm.sh + - post-wm.sh + - email-monitor.sh + - realtime-monitor.sh + tags: + - deploy + - api-server + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/deploy_file_server.yml b/dev-tools/ansible/roles/airavata_services/tasks/deploy_file_server.yml new file mode 100644 index 0000000000..99bb583f85 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/deploy_file_server.yml @@ -0,0 +1,78 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Remove old File Server deployment if exists + file: + path: "{{ file_server_dir }}" + state: absent + tags: + - deploy + - file-server + +- name: Extract File Server distribution + unarchive: + src: "{{ airavata_source_dir }}/distribution/apache-airavata-file-server-{{ airavata_version }}.tar.gz" + dest: "{{ deployment_dir }}" + remote_src: yes + tags: + - deploy + - file-server + +- name: Create configuration directory for File Server + file: + path: "{{ file_server_dir }}/conf" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + tags: + - deploy + - file-server + +- name: Deploy application.properties for File Server + template: + src: application-file-server.properties.j2 + dest: "{{ file_server_dir }}/conf/application.properties" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - file-server + +- name: Deploy log4j2.xml for File Server + template: + src: log4j2.xml.j2 + dest: "{{ file_server_dir }}/conf/log4j2.xml" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - file-server + +- name: Set executable permissions on File Server script + file: + path: "{{ file_server_dir }}/bin/file-service.sh" + mode: "0755" + tags: + - deploy + - file-server + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/deploy_research_service.yml b/dev-tools/ansible/roles/airavata_services/tasks/deploy_research_service.yml new file mode 100644 index 0000000000..132f3e715c --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/deploy_research_service.yml @@ -0,0 +1,78 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Remove old Research Service deployment if exists + file: + path: "{{ research_service_dir }}" + state: absent + tags: + - deploy + - research-service + +- name: Extract Research Service distribution + unarchive: + src: "{{ airavata_source_dir }}/distribution/apache-airavata-research-service-{{ airavata_version }}.tar.gz" + dest: "{{ deployment_dir }}" + remote_src: yes + tags: + - deploy + - research-service + +- name: Create configuration directory for Research Service + file: + path: "{{ research_service_dir }}/conf" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + tags: + - deploy + - research-service + +- name: Deploy application.yml for Research Service + template: + src: application-research-service.yml.j2 + dest: "{{ research_service_dir }}/conf/application.yml" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - research-service + +- name: Deploy log4j2.xml for Research Service + template: + src: log4j2.xml.j2 + dest: "{{ research_service_dir }}/conf/log4j2.xml" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - research-service + +- name: Set executable permissions on Research Service script + file: + path: "{{ research_service_dir }}/bin/research-service.sh" + mode: "0755" + tags: + - deploy + - research-service + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/deploy_restproxy.yml b/dev-tools/ansible/roles/airavata_services/tasks/deploy_restproxy.yml new file mode 100644 index 0000000000..0aeaa94a0e --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/deploy_restproxy.yml @@ -0,0 +1,78 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Remove old REST Proxy deployment if exists + file: + path: "{{ restproxy_dir }}" + state: absent + tags: + - deploy + - restproxy + +- name: Extract REST Proxy distribution + unarchive: + src: "{{ airavata_source_dir }}/distribution/apache-airavata-restproxy-{{ airavata_version }}.tar.gz" + dest: "{{ deployment_dir }}" + remote_src: yes + tags: + - deploy + - restproxy + +- name: Create configuration directory for REST Proxy + file: + path: "{{ restproxy_dir }}/conf" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + tags: + - deploy + - restproxy + +- name: Deploy application.properties for REST Proxy + template: + src: application-restproxy.properties.j2 + dest: "{{ restproxy_dir }}/conf/application.properties" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - restproxy + +- name: Deploy log4j2.xml for REST Proxy + template: + src: log4j2.xml.j2 + dest: "{{ restproxy_dir }}/conf/log4j2.xml" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0644" + tags: + - deploy + - restproxy + +- name: Set executable permissions on REST Proxy script + file: + path: "{{ restproxy_dir }}/bin/restproxy.sh" + mode: "0755" + tags: + - deploy + - restproxy + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/generate_keystore.yml b/dev-tools/ansible/roles/airavata_services/tasks/generate_keystore.yml new file mode 100644 index 0000000000..306afaa003 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/generate_keystore.yml @@ -0,0 +1,149 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +# Generate keystore from Let's Encrypt certificates +# This must run AFTER Let's Encrypt role has generated certificates + +- name: Set Let's Encrypt certificate directory + set_fact: + letsencrypt_cert_dir: "/etc/letsencrypt/live/{{ api_server_public_hostname | default('localhost') }}" + +- name: Check if Let's Encrypt certificate exists + stat: + path: "{{ letsencrypt_cert_dir }}/fullchain.pem" + register: letsencrypt_cert_check + become: yes + become_user: root + +- name: Fail if Let's Encrypt certificate not found + fail: + msg: "Let's Encrypt certificate not found at {{ letsencrypt_cert_dir }}/fullchain.pem. Run Let's Encrypt role first." + when: not letsencrypt_cert_check.stat.exists + +- name: Create temporary directory for keystore generation + file: + path: "/tmp/keystore-generation" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0700" + become: yes + when: letsencrypt_cert_check.stat.exists + +- name: Create keystores directory + file: + path: "{{ api_server_dir }}/conf/keystores" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0700" + become: yes + +- name: Copy Let's Encrypt certificates to temp directory (for user access) + copy: + src: "{{ item }}" + dest: "/tmp/keystore-generation/{{ item | basename }}" + remote_src: yes + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0600" + loop: + - "{{ letsencrypt_cert_dir }}/fullchain.pem" + - "{{ letsencrypt_cert_dir }}/privkey.pem" + become: yes + become_user: root + when: letsencrypt_cert_check.stat.exists + +- name: Detect Java home if not already set + shell: find /usr/lib/jvm -name java -type f -path "*/bin/java" 2>/dev/null | head -1 | xargs dirname | xargs dirname + register: detected_java_home + changed_when: false + failed_when: false + when: java_home is not defined + +- name: Set Java home from detected path + set_fact: + java_home: "{{ detected_java_home.stdout }}" + when: java_home is not defined and detected_java_home.stdout != "" + +- name: Set default Java home if detection failed + set_fact: + java_home: "{{ '/usr/lib/jvm/java-17-openjdk-amd64' if ansible_os_family == 'Debian' else '/usr/lib/jvm/java-17' }}" + when: java_home is not defined + +- name: Generate AES-256 key for credential encryption + shell: | + keytool -genseckey -alias airavata -keyalg AES -keysize 256 \ + -keystore /tmp/keystore-generation/aes.p12 \ + -storepass "{{ keystore_password }}" + args: + chdir: "/tmp/keystore-generation" + creates: "/tmp/keystore-generation/aes.p12" + become: yes + become_user: "{{ deploy_user }}" + environment: + JAVA_HOME: "{{ java_home }}" + PATH: "{{ java_home }}/bin:{{ ansible_env.PATH }}" + when: letsencrypt_cert_check.stat.exists + +- name: Generate keystore from Let's Encrypt certificates + shell: | + openssl pkcs12 -export -name tls \ + -out /tmp/keystore-generation/airavata.p12 \ + -passout pass:{{ keystore_password }} \ + -in /tmp/keystore-generation/fullchain.pem \ + -inkey /tmp/keystore-generation/privkey.pem && \ + keytool -importkeystore \ + -srckeystore /tmp/keystore-generation/aes.p12 \ + -destkeystore /tmp/keystore-generation/airavata.p12 \ + -srcstorepass {{ keystore_password }} \ + -deststorepass {{ keystore_password }} \ + -noprompt && \ + cp /tmp/keystore-generation/airavata.p12 {{ api_server_dir }}/conf/keystores/{{ vault_keystore_file }} && \ + cp /tmp/keystore-generation/airavata.p12 {{ api_server_dir }}/conf/keystores/airavata.p12 + args: + chdir: "/tmp/keystore-generation" + become: yes + become_user: "{{ deploy_user }}" + environment: + JAVA_HOME: "{{ java_home }}" + PATH: "{{ java_home }}/bin:{{ ansible_env.PATH }}" + when: letsencrypt_cert_check.stat.exists + +- name: Set proper permissions on keystore files + file: + path: "{{ api_server_dir }}/conf/keystores/{{ item }}" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0600" + loop: + - "{{ vault_keystore_file }}" + - "airavata.p12" + become: yes + when: letsencrypt_cert_check.stat.exists + +- name: Clean up temporary directory + file: + path: "/tmp/keystore-generation" + state: absent + become: yes + when: always | default(true) + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/main.yml b/dev-tools/ansible/roles/airavata_services/tasks/main.yml new file mode 100644 index 0000000000..1b68a05f37 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/main.yml @@ -0,0 +1,49 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Stop all Airavata services before deployment + include_tasks: stop_services.yml + when: not skip_stop_services | default(false) + tags: + - deploy + - stop + +- name: Include service deployment tasks + include_tasks: deploy_api_server.yml + +- name: Include agent service deployment tasks + include_tasks: deploy_agent_service.yml + +- name: Include research service deployment tasks + include_tasks: deploy_research_service.yml + +- name: Include file server deployment tasks + include_tasks: deploy_file_server.yml + +- name: Include REST proxy deployment tasks + include_tasks: deploy_restproxy.yml + +- name: Start all Airavata services + include_tasks: start_services.yml + tags: + - start + - deploy + diff --git a/dev-tools/ansible/roles/airavata_services/tasks/start_services.yml b/dev-tools/ansible/roles/airavata_services/tasks/start_services.yml new file mode 100644 index 0000000000..33ca2941eb --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/start_services.yml @@ -0,0 +1,152 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +- name: Start API Server orchestrator + shell: '{{ api_server_dir }}/bin/orchestrator.sh -d start api-orch' + args: + chdir: '{{ api_server_dir }}' + tags: + - start + - api-server + +- name: Wait for API Server orchestrator to be ready + wait_for: + port: '{{ orchestrator_server_port }}' + host: localhost + delay: 5 + timeout: 60 + tags: + - start + - api-server + +- name: Start API Server controller + shell: '{{ api_server_dir }}/bin/controller.sh -d start' + args: + chdir: '{{ api_server_dir }}' + tags: + - start + - api-server + +- name: Start API Server participant + shell: '{{ api_server_dir }}/bin/participant.sh -d start' + args: + chdir: '{{ api_server_dir }}' + tags: + - start + - api-server + +- name: Start API Server email-monitor + shell: '{{ api_server_dir }}/bin/email-monitor.sh -d start' + args: + chdir: '{{ api_server_dir }}' + tags: + - start + - api-server + +- name: Start API Server realtime-monitor + shell: '{{ api_server_dir }}/bin/realtime-monitor.sh -d start' + args: + chdir: '{{ api_server_dir }}' + tags: + - start + - api-server + +- name: Start API Server pre-wm + shell: '{{ api_server_dir }}/bin/pre-wm.sh -d start' + args: + chdir: '{{ api_server_dir }}' + tags: + - start + - api-server + +- name: Start API Server post-wm + shell: '{{ api_server_dir }}/bin/post-wm.sh -d start' + args: + chdir: '{{ api_server_dir }}' + tags: + - start + - api-server + +- name: Wait for API Server to start + wait_for: + timeout: 30 + tags: + - start + - api-server + +- name: Start Agent Service + shell: '{{ agent_service_dir }}/bin/agent-service.sh -d start' + args: + chdir: '{{ agent_service_dir }}' + tags: + - start + - agent-service + +- name: Wait for Agent Service to start + wait_for: + timeout: 30 + tags: + - start + - agent-service + +- name: Start Research Service + shell: '{{ research_service_dir }}/bin/research-service.sh -d start' + args: + chdir: '{{ research_service_dir }}' + tags: + - start + - research-service + +- name: Wait for Research Service to start + wait_for: + timeout: 30 + tags: + - start + - research-service + +- name: Start File Server + shell: '{{ file_server_dir }}/bin/file-service.sh -d start' + args: + chdir: '{{ file_server_dir }}' + tags: + - start + - file-server + +- name: Wait for File Server to start + wait_for: + timeout: 30 + tags: + - start + - file-server + +- name: Start REST Proxy + shell: '{{ restproxy_dir }}/bin/restproxy.sh -d start' + args: + chdir: '{{ restproxy_dir }}' + tags: + - start + - restproxy + +- name: Wait for REST Proxy to start + wait_for: + timeout: 30 + tags: + - start + - restproxy \ No newline at end of file diff --git a/dev-tools/ansible/roles/airavata_services/tasks/stop_services.yml b/dev-tools/ansible/roles/airavata_services/tasks/stop_services.yml new file mode 100644 index 0000000000..c8a8c0a9b7 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/tasks/stop_services.yml @@ -0,0 +1,287 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: Check if API Server binary exists + stat: + path: "{{ api_server_dir }}/bin/orchestrator.sh" + register: api_server_exists + tags: + - stop + +- name: Stop API Server orchestrator if exists + shell: "{{ api_server_dir }}/bin/orchestrator.sh -d stop api-orch" + when: api_server_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Stop API Server controller if exists + shell: "{{ api_server_dir }}/bin/controller.sh -d stop" + when: api_server_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Stop API Server participant if exists + shell: "{{ api_server_dir }}/bin/participant.sh -d stop" + when: api_server_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Stop API Server pre-wm if exists + shell: "{{ api_server_dir }}/bin/pre-wm.sh -d stop" + when: api_server_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Stop API Server post-wm if exists + shell: "{{ api_server_dir }}/bin/post-wm.sh -d stop" + when: api_server_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Stop API Server email-monitor if exists + shell: "{{ api_server_dir }}/bin/email-monitor.sh -d stop" + when: api_server_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Stop API Server realtime-monitor if exists + shell: "{{ api_server_dir }}/bin/realtime-monitor.sh -d stop" + when: api_server_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Check if Agent Service binary exists + stat: + path: "{{ agent_service_dir }}/bin/agent-service.sh" + register: agent_service_exists + tags: + - stop + +- name: Stop Agent Service if exists + shell: "{{ agent_service_dir }}/bin/agent-service.sh -d stop" + when: agent_service_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Check if Research Service binary exists + stat: + path: "{{ research_service_dir }}/bin/research-service.sh" + register: research_service_exists + tags: + - stop + +- name: Stop Research Service if exists + shell: "{{ research_service_dir }}/bin/research-service.sh -d stop" + when: research_service_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Check if File Server binary exists + stat: + path: "{{ file_server_dir }}/bin/file-service.sh" + register: file_server_exists + tags: + - stop + +- name: Stop File Server if exists + shell: "{{ file_server_dir }}/bin/file-service.sh -d stop" + when: file_server_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Check if REST Proxy binary exists + stat: + path: "{{ restproxy_dir }}/bin/restproxy.sh" + register: restproxy_exists + tags: + - stop + +- name: Stop REST Proxy if exists + shell: "{{ restproxy_dir }}/bin/restproxy.sh -d stop" + when: restproxy_exists.stat.exists + tags: + - stop + ignore_errors: yes + +- name: Wait for services to fully stop + wait_for: + timeout: 30 + tags: + - stop + +- name: Define Airavata service ports + set_fact: + airavata_ports: + - port: "{{ api_server_port }}" + service: "API Server" + - port: "{{ orchestrator_server_port }}" + service: "Orchestrator" + - port: "{{ registry_server_port }}" + service: "Registry" + - port: "{{ sharing_registry_port }}" + service: "Sharing Registry" + - port: "{{ cred_store_port }}" + service: "Credential Store" + - port: "{{ profile_service_port }}" + service: "Profile Service" + - port: "{{ agent_service_server_port }}" + service: "Agent Service" + - port: "{{ research_service_server_port }}" + service: "Research Service" + - port: "{{ file_server_port }}" + service: "File Server" + - port: "{{ restproxy_port }}" + service: "REST Proxy" + - port: "{{ participant_monitoring_port }}" + service: "Participant Monitoring" + - port: "{{ api_server_monitoring_port }}" + service: "API Server Monitoring" + - port: "{{ pre_wm_monitoring_port }}" + service: "Pre-WM Monitoring" + - port: "{{ post_wm_monitoring_port }}" + service: "Post-WM Monitoring" + tags: + - stop + +- name: Check which ports are still in use after stopping services + shell: | + if command -v ss >/dev/null 2>&1; then + ss -tuln | grep -E '^[^ ]* *[^ ]* *[^ ]* *[^ ]* *[^ ]* *LISTEN' | awk '{print $5}' | sed 's/.*://' | sort -u + elif command -v netstat >/dev/null 2>&1; then + netstat -tuln | grep LISTEN | awk '{print $4}' | sed 's/.*://' | sort -u + else + echo "" + fi + register: ports_in_use + changed_when: false + tags: + - stop + +- name: Find processes still using Airavata ports (graceful kill) + shell: | + PORT="{{ item.port }}" + SERVICE="{{ item.service }}" + PID="" + + if command -v ss >/dev/null 2>&1; then + # Try with sed (more portable than grep -P) + PID=$(ss -tulpn 2>/dev/null | grep ":${PORT} " | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | head -1) + elif command -v netstat >/dev/null 2>&1; then + PID=$(netstat -tulpn 2>/dev/null | grep ":${PORT} " | awk '{print $7}' | cut -d'/' -f1 | head -1) + fi + + if [ -z "$PID" ] || [ "$PID" == "-" ]; then + if command -v lsof >/dev/null 2>&1; then + PID=$(lsof -ti:${PORT} 2>/dev/null | head -1) + fi + fi + + if [ -n "$PID" ] && [ "$PID" != "-" ] && [ "$PID" != "" ]; then + echo "Found process $PID using port $PORT ($SERVICE), attempting graceful kill..." + kill -TERM "$PID" 2>/dev/null || true + sleep 2 + if kill -0 "$PID" 2>/dev/null; then + echo "Process $PID still running, forcing kill..." + kill -KILL "$PID" 2>/dev/null || true + sleep 1 + fi + echo "Process $PID terminated" + else + echo "No process found using port $PORT ($SERVICE)" + fi + loop: "{{ airavata_ports }}" + when: ports_in_use.stdout != "" and item.port|string in ports_in_use.stdout_lines + ignore_errors: yes + tags: + - stop + +- name: Wait for ports to be freed + wait_for: + timeout: 10 + tags: + - stop + +- name: Verify all Airavata ports are free + shell: | + PORT="{{ item.port }}" + SERVICE="{{ item.service }}" + if command -v ss >/dev/null 2>&1; then + IN_USE=$(ss -tuln | grep -c ":${PORT} " || echo "0") + elif command -v netstat >/dev/null 2>&1; then + IN_USE=$(netstat -tuln 2>/dev/null | grep -c ":${PORT} " || echo "0") + elif command -v lsof >/dev/null 2>&1; then + IN_USE=$(lsof -ti:${PORT} 2>/dev/null | wc -l || echo "0") + else + IN_USE="0" + fi + + if [ "$IN_USE" != "0" ]; then + echo "WARNING: Port ${PORT} (${SERVICE}) is still in use" + exit 1 + else + echo "Port ${PORT} (${SERVICE}) is free" + fi + loop: "{{ airavata_ports }}" + failed_when: false + changed_when: false + tags: + - stop + +- name: Define Airavata service log directories + set_fact: + airavata_log_dirs: + - { service: "API Server", path: "{{ api_server_dir }}/logs" } + - { service: "Agent Service", path: "{{ agent_service_dir }}/logs" } + - { service: "Research Service", path: "{{ research_service_dir }}/logs" } + - { service: "File Server", path: "{{ file_server_dir }}/logs" } + - { service: "REST Proxy", path: "{{ restproxy_dir }}/logs" } + when: archive_logs_on_stop | bool + +- name: Check log directories + stat: + path: "{{ item.path }}" + register: airavata_log_dir_stats + loop: "{{ airavata_log_dirs }}" + when: archive_logs_on_stop | bool + +- name: Archive Airavata service logs + shell: | + ts=$(date +%Y%m%d%H%M%S) + dest_dir="{{ item.item.path }}/archive/${ts}" + mkdir -p "${dest_dir}" + find "{{ item.item.path }}" -maxdepth 1 -type f -name "*.log*" -print -exec mv {} "${dest_dir}/" \; + args: + executable: /bin/bash + loop: "{{ airavata_log_dir_stats.results }}" + when: archive_logs_on_stop | bool and item.stat.exists + changed_when: true + tags: + - stop diff --git a/dev-tools/ansible/roles/airavata_services/templates/airavata-server.properties.j2 b/dev-tools/ansible/roles/airavata_services/templates/airavata-server.properties.j2 new file mode 100644 index 0000000000..33dce0027a --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/templates/airavata-server.properties.j2 @@ -0,0 +1,276 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +########################################################################### +# +# This properties file provides configuration for all Airavata Services: +# +########################################################################### + +########################################################################### +# API Server Registry DB Configuration +########################################################################### +registry.jdbc.driver={{ registry_jdbc_driver }} +registry.jdbc.url={{ registry_jdbc_url }} +registry.jdbc.user={{ registry_jdbc_user }} +registry.jdbc.password={{ registry_jdbc_password }} +validationQuery=SELECT 1 from CONFIGURATION + +########################################################################### +# Application Catalog DB Configuration +########################################################################### +appcatalog.jdbc.driver={{ appcatalog_jdbc_driver }} +appcatalog.jdbc.url={{ appcatalog_jdbc_url }} +appcatalog.jdbc.user={{ appcatalog_jdbc_user }} +appcatalog.jdbc.password={{ appcatalog_jdbc_password }} +appcatalog.validationQuery=SELECT 1 from CONFIGURATION + +########################################################################## +# Replica Catalog DB Configuration +########################################################################### +replicacatalog.jdbc.driver={{ replicacatalog_jdbc_driver }} +replicacatalog.jdbc.url={{ replicacatalog_jdbc_url }} +replicacatalog.jdbc.user={{ replicacatalog_jdbc_user }} +replicacatalog.jdbc.password={{ replicacatalog_jdbc_password }} +replicacatalog.validationQuery=SELECT 1 from CONFIGURATION + +########################################################################### +# Workflow Catalog DB Configuration +########################################################################### +workflowcatalog.jdbc.driver={{ workflowcatalog_jdbc_driver }} +workflowcatalog.jdbc.url={{ workflowcatalog_jdbc_url }} +workflowcatalog.jdbc.user={{ workflowcatalog_jdbc_user }} +workflowcatalog.jdbc.password={{ workflowcatalog_jdbc_password }} +workflowcatalog.validationQuery=SELECT 1 from CONFIGURATION + +########################################################################### +# Sharing Catalog DB Configuration +########################################################################### +sharingcatalog.jdbc.driver={{ sharingcatalog_jdbc_driver }} +sharingcatalog.jdbc.url={{ sharingcatalog_jdbc_url }} +sharingcatalog.jdbc.user={{ sharingcatalog_jdbc_user }} +sharingcatalog.jdbc.password={{ sharingcatalog_jdbc_password }} +sharingcatalog.validationQuery=SELECT 1 from CONFIGURATION + +########################################################################### +# Generic Server Configurations +########################################################################### +enable.sharing={{ enable_sharing }} + +########################################################################### +# Sharing Registry Server Configuration +########################################################################### +sharing_server={{ sharing_server_class }} +sharing.registry.server.host={{ sharing_registry_bind_host | default(sharing_registry_server_host) }} +sharing.registry.server.port={{ sharing_registry_port }} + +########################################################################### +# Registry Server Configurations +########################################################################### +regserver={{ regserver_class }} +regserver.server.host={{ registry_bind_host | default(regserver_server_host) }} +regserver.server.port={{ registry_server_port }} +regserver.server.min.threads={{ orchestrator_server_min_threads }} + +default.registry.user={{ default_registry_user }} +default.registry.password={{ default_registry_password }} +default.registry.gateway={{ default_registry_gateway }} +default.registry.oauth.client.id={{ default_registry_oauth_client_id }} +default.registry.oauth.client.secret={{ default_registry_oauth_client_secret }} +super.tenant.gatewayId={{ super_tenant_gatewayId }} + +########################################################################### +# API Server Configurations +########################################################################### +apiserver.class={{ apiserver_class }} +apiserver.host={{ api_server_bind_host | default(api_server_host) }} +apiserver.port={{ api_server_port }} +api.server.monitoring.enabled={{ api_server_monitoring_enabled }} +api.server.monitoring.host={{ api_server_monitoring_host }} +api.server.monitoring.port={{ api_server_monitoring_port }} + +########################################################################### +# Orchestrator Server Configurations +########################################################################### +orchestrator={{ orchestrator_class }} +orchestrator.server.host={{ orchestrator_bind_host | default(orchestrator_server_host) }} +orchestrator.server.port={{ orchestrator_server_port }} +orchestrator.server.min.threads={{ orchestrator_server_min_threads }} + +job.validators={{ job_validators }} +enable.validation={{ enable_validation }} +host.scheduler={{ host_scheduler }} + +########################################################################### +# Job Monitor Configurations +########################################################################### +job.notification.enable={{ job_notification_enable }} +#Provide comma separated email ids as a string if more than one +job.notification.emailids={{ job_notification_emailids }} +job.status.publish.endpoint={{ job_status_publish_endpoint }} + +########################################################################### +# Credential Store module Configuration +########################################################################### +credential.store.keystore.url={{ credential_store_keystore_url }} +credential.store.keystore.alias={{ credential_store_keystore_alias }} +credential.store.keystore.password={{ credential_store_keystore_password }} +credential.store.jdbc.url={{ credential_store_jdbc_url }} +credential.store.jdbc.user={{ credential_store_jdbc_user }} +credential.store.jdbc.password={{ credential_store_jdbc_password }} +credential.store.jdbc.driver={{ credential_store_jdbc_driver }} +credential.store.server.host={{ cred_store_server_bind_host | default(cred_store_server_host) }} +credential.store.server.port={{ cred_store_port }} +credential.store.class={{ credential_store_class }} +credential.store.jdbc.validationQuery={{ credential_store_validation_query }} + +########################################################################### +# AMQP Notification Configuration +########################################################################### +rabbitmq.broker.url={{ rabbitmq_broker_url }} +experiment.launch.queue={{ experiment_launch_queue }} +rabbitmq.status.exchange.name={{ rabbitmq_status_exchange_name }} +rabbitmq.process.exchange.name={{ rabbitmq_process_exchange_name }} +rabbitmq.experiment.exchange.name={{ rabbitmq_experiment_exchange_name }} +durable.queue={{ durable_queue }} +prefetch.count={{ prefetch_count }} + +########################################################################### +# Zookeeper Server Configuration +########################################################################### +embedded.zk={{ embedded_zk }} +zookeeper.server.connection={{ zookeeper_connection }} + +######################################################################## +## API Security Configuration +######################################################################## +security.manager.class={{ security_manager_class }} +TLS.enabled={{ TLS_enabled }} +TLS.client.timeout={{ TLS_client_timeout }} +keystore.path={{ keystore_path }} +keystore.password={{ keystore_password }} +authz.cache.enabled={{ authz_cache_enabled }} +authz.cache.manager.class={{ authz_cache_manager_class }} +in.memory.cache.size={{ in_memory_cache_size }} + +########################################################################### +# Profile Service Configuration +########################################################################### +profile.service.server.host={{ profile_service_bind_host }} +profile.service.server.port={{ profile_service_port }} +profile_service.class={{ profile_service_class }} +# MariaDB properties +profile.service.jdbc.url={{ profile_service_jdbc_url }} +profile.service.jdbc.user={{ profile_service_jdbc_user }} +profile.service.jdbc.password={{ profile_service_jdbc_password }} +profile.service.jdbc.driver={{ profile_service_jdbc_driver }} +profile.service.validationQuery={{ profile_service_validation_query }} + +########################################################################### +# Iam Admin services Configuration +########################################################################### +iam.server.url={{ iam_server_url }} +iam.server.super.admin.username={{ iam_admin_username }} +iam.server.super.admin.password={{ iam_admin_password }} + +########################################################################### +# DB Event Manager Runner +########################################################################### +db_event_manager.class={{ db_event_manager_class }} + +########################################################################### +# Job Execution Engine properties +########################################################################### +helix.cluster.name={{ helix_cluster_name }} +helix.controller.name={{ helix_controller_name }} +helix.participant.name={{ helix_participant_name }} +participant.monitoring.enabled={{ participant_monitoring_enabled }} +participant.monitoring.host={{ participant_monitoring_host }} +participant.monitoring.port={{ participant_monitoring_port }} + +########################################################################### +# Job Monitor related properties +########################################################################### +enable.realtime.monitor={{ enable_realtime_monitor }} +realtime.monitor.broker.consumer.group={{ realtime_monitor_broker_consumer_group }} +realtime.monitor.broker.topic={{ realtime_monitor_broker_topic }} + +job.monitor.broker.consumer.group={{ job_monitor_broker_consumer_group }} +job.monitor.broker.topic={{ job_monitor_broker_topic }} +job.monitor.broker.publisher.id={{ job_monitor_broker_publisher_id }} +job.monitor.email.publisher.id={{ job_monitor_email_publisher_id }} +job.monitor.realtime.publisher.id={{ job_monitor_realtime_publisher_id }} + +email.based.monitor.host={{ email_based_monitor_host }} +email.based.monitor.store.protocol={{ email_based_monitor_store_protocol }} +email.based.monitor.folder.name={{ email_based_monitor_folder_name }} +email.expiration.minutes={{ email_expiration_minutes }} +email.based.monitoring.period={{ email_based_monitoring_period }} +email.based.monitor.address={{ email_based_monitor_address }} +email.based.monitor.password={{ email_based_monitor_password }} + +kafka.broker.url={{ kafka_broker_url }} +local.data.location={{ local_data_location }} + +########################################################################### +# ThriftClientPool Configuration +########################################################################### +thrift.client.pool.abandoned.removal.enabled={{ thrift_client_pool_abandoned_removal_enabled }} +thrift.client.pool.abandoned.removal.logged={{ thrift_client_pool_abandoned_removal_logged }} + +########################################################################### +# Pre-workflow Configuration +########################################################################### +pre.workflow.manager.loadbalance.clusters={{ pre_workflow_manager_loadbalance_clusters }} +pre.workflow.manager.monitoring.enabled={{ pre_workflow_manager_monitoring_enabled }} +pre.workflow.manager.monitoring.host={{ pre_workflow_manager_monitoring_host }} +pre.workflow.manager.monitoring.port={{ pre_wm_monitoring_port }} +pre.workflow.manager.name={{ pre_workflow_manager_name }} + +########################################################################### +# Post-workflow Configuration +########################################################################### +post.workflow.manager.loadbalance.clusters={{ post_workflow_manager_loadbalance_clusters }} +post.workflow.manager.monitoring.enabled={{ post_workflow_manager_monitoring_enabled }} +post.workflow.manager.monitoring.host={{ post_workflow_manager_monitoring_host }} +post.workflow.manager.monitoring.port={{ post_wm_monitoring_port }} +post.workflow.manager.name={{ post_workflow_manager_name }} + +########################################################################### +# Parser-workflow Configuration +########################################################################### +data.parser.delete.container={{ data_parser_delete_container }} +data.parser.broker.consumer.group={{ data_parser_broker_consumer_group }} +data.parser.topic={{ data_parser_topic }} +data.parser.storage.resource.id={{ data_parser_storage_resource_id }} + +########################################################################### +# Metascheduler And Compute Resource Monitoring Configuration +########################################################################### +cluster.status.monitoring.enable={{ cluster_status_monitoring_enable }} +# cluster.status.monitoring.repeat.time=18000 +metaschedluer.job.scanning.enable={{ metaschedluer_job_scanning_enable }} +data.analyzer.job.scanning.enable={{ data_analyzer_job_scanning_enable }} + +########################################################################### +# Data Staging Task Level Configurations +########################################################################### +enable.streaming.transfer={{ enable_streaming_transfer }} + diff --git a/dev-tools/ansible/roles/airavata_services/templates/application-agent-service.yml.j2 b/dev-tools/ansible/roles/airavata_services/templates/application-agent-service.yml.j2 new file mode 100644 index 0000000000..1d06ef8489 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/templates/application-agent-service.yml.j2 @@ -0,0 +1,63 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +grpc: + server: + host: {{ agent_service_grpc_host }} + port: {{ agent_service_grpc_port }} + max-inbound-message-size: {{ agent_service_grpc_max_inbound_message_size }} + +server: + port: {{ agent_service_server_port }} + address: {{ agent_service_server_address }} + +spring: + servlet: + multipart: + max-file-size: {{ agent_service_max_file_size }} + max-request-size: {{ agent_service_max_request_size }} + datasource: + url: "{{ agent_service_datasource_url }}" + username: "{{ agent_service_datasource_username | default(db_user | default('airavata')) }}" + password: "{{ agent_service_datasource_password | default(db_password | default('')) }}" + driver-class-name: org.mariadb.jdbc.Driver + hikari: + pool-name: {{ agent_service_pool_name }} + leak-detection-threshold: {{ agent_service_leak_detection_threshold }} + jpa: + hibernate: + ddl-auto: {{ agent_service_ddl_auto }} + open-in-view: false + +airavata: + server: + url: {{ agent_service_airavata_url }} + port: {{ agent_service_airavata_port }} + secure: {{ agent_service_airavata_secure }} + storageResourceId: {{ agent_service_storage_resource_id }} + storagePath: {{ agent_service_storage_path }} + cluster: + applicationInterfaceId: {{ agent_service_application_interface_id }} + tunnel: + serverHost: {{ tunnel_server_host }} + serverPort: {{ tunnel_server_port }} + serverToken: {{ tunnel_server_token }} + serverApiUrl: {{ tunnel_server_api_url }} + diff --git a/dev-tools/ansible/roles/airavata_services/templates/application-file-server.properties.j2 b/dev-tools/ansible/roles/airavata_services/templates/application-file-server.properties.j2 new file mode 100644 index 0000000000..4a8e80afc7 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/templates/application-file-server.properties.j2 @@ -0,0 +1,34 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +spring.servlet.multipart.max-file-size={{ file_server_max_file_size }} +spring.servlet.multipart.max-request-size={{ file_server_max_request_size }} +spring.servlet.multipart.file-size-threshold={{ file_server_file_size_threshold }} + +# files storage location (stores all files uploaded via REST API) +storage.location={{ file_server_storage_location }} + +regserver.server.host={{ regserver_server_host }} +regserver.server.port={{ registry_server_port }} +credential.store.server.host={{ credential_store_server_host }} +credential.store.server.port={{ cred_store_port }} + +server.port={{ file_server_port }} + diff --git a/dev-tools/ansible/roles/airavata_services/templates/application-research-service.yml.j2 b/dev-tools/ansible/roles/airavata_services/templates/application-research-service.yml.j2 new file mode 100644 index 0000000000..e3daeb1d76 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/templates/application-research-service.yml.j2 @@ -0,0 +1,76 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +grpc: + server: + port: {{ research_service_grpc_port }} + +server: + port: {{ research_service_server_port }} + address: {{ research_service_server_address }} + +airavata: + research-hub: + url: {{ research_hub_url }} + dev-user: "{{ research_hub_dev_user }}" + adminApiKey: "{{ research_hub_admin_api_key }}" + limit: {{ research_hub_limit }} + research-portal: + dev-url: {{ research_portal_dev_url }} + url: {{ research_portal_url }} + + openid: + url: "{{ openid_url }}" + + user-profile: + server: + url: {{ user_profile_server_url }} + port: {{ user_profile_server_port }} + +spring: + servlet: + multipart: + max-file-size: {{ research_service_max_file_size }} + max-request-size: {{ research_service_max_request_size }} + datasource: + url: "{{ research_service_datasource_url }}" + username: "{{ research_service_datasource_username | default(db_user | default('airavata')) }}" + password: "{{ research_service_datasource_password | default(db_password | default('')) }}" + driver-class-name: org.mariadb.jdbc.Driver + hikari: + pool-name: {{ research_service_pool_name }} + leak-detection-threshold: {{ research_service_leak_detection_threshold }} + jpa: + hibernate: + ddl-auto: {{ research_service_ddl_auto }} + open-in-view: false + +springdoc: + api-docs: + enabled: {{ springdoc_api_docs_enabled }} + swagger-ui: + path: {{ springdoc_swagger_ui_path }} + operationsSorter: {{ springdoc_swagger_ui_operations_sorter }} + tagsSorter: {{ springdoc_swagger_ui_tags_sorter }} + doc-expansion: {{ springdoc_swagger_ui_doc_expansion }} + oauth: + use-pkce-with-authorization-code-grant: {{ springdoc_swagger_ui_oauth_use_pkce }} + client-id: {{ springdoc_swagger_ui_oauth_client_id }} + diff --git a/dev-tools/ansible/roles/airavata_services/templates/application-restproxy.properties.j2 b/dev-tools/ansible/roles/airavata_services/templates/application-restproxy.properties.j2 new file mode 100644 index 0000000000..0659c934e1 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/templates/application-restproxy.properties.j2 @@ -0,0 +1,23 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +restproxy.broker.url={{ restproxy_broker_url }} +server.port={{ restproxy_port }} + diff --git a/dev-tools/ansible/roles/airavata_services/templates/email-config.yml.j2 b/dev-tools/ansible/roles/airavata_services/templates/email-config.yml.j2 new file mode 100644 index 0000000000..e361cfd320 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/templates/email-config.yml.j2 @@ -0,0 +1,114 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +config: + resources: + - jobManagerType: PBS + emailParser: org.apache.airavata.monitor.email.parser.PBSEmailParser + resourceEmailAddresses: + - pbsconsult@sdsc.edu # gordon + - adm@trident.bigred2.uits.iu.edu # Bigred2 + - root # Bigred2 + - root # alamo + - root #karst + - root #mason + - smic3 # philip.hpc.lsu + - adm@jetstream-cloud.org + - adm #supermic + - root #alamo + - root #carbonate + - adm # Mick cluster at LSU for CSBG-LSU + - GW77 Job Emails #gw77 email to fix email issues + + - jobManagerType: SLURM + emailParser: org.apache.airavata.monitor.email.parser.SLURMEmailParser + resourceEmailAddresses: + - SDSC Admin # comet + - slurm@batch1.stampede.tacc.utexas.edu # stampede + - SDSC Admin # comet new + - slurm@comet-fe4.sdsc.edu + - Slurm # bridges + - Slurm #Bridges + - Slurm Daemon # OU Schooner + - slurm@lnet28.stampede.tacc.utexas.edu # stampede2 + - Slurm service account # Utah Ember + - SLURM workload manager # JS Mark Cluster + - super user # LS5 new email after Nov 5th 2018 + - root@master.ls5.tacc.utexas.edu #LS5 old email + - batch-jsc@fz-juelich.de # Jureca Email + - Slurm + - slurm@jetstream-cloud.org + - slurm@slurm-example.novalocal + - slurm@tutorial-headnode.novalocal # Jetstream Ultrascan static cluster with 10 nodes + - slurm@batch1.stampede2.tacc.utexas.edu #Stampede2 + - slurm@sra-master.jetstreamlocal #Searching SRA + - Slurm Admin #GSU cluster + - slurm@head.cluster #USD cluster + - slurm@js-169-158.jetstream-cloud.org + - slurm@joker.nmsu.edu + - SLURM resource manager #Bigdawg + - SLURM resource manager #InterACTWEL Jetstream + - slurm@zoar #R System cluster + - GW77 Job Emails #gw77 email to fix email issues + - SLURM resource manager #seagrid elastic cluster + - SLURM resource manager # EPW Jetstream slurm cluster + - SLURM resource manager #Distant reader + - slurm@thunder.jacks.local #RT at SDSU + - Slurm on Mio # MIO from Mines + - SLURM resource manager # Jetstream Helix cluster + - SLURM resource manager + - slurm@sdsc.edu + - super user # Bigred3 + - SLURM resource manager + - SLURM resource manager + - Slurm + - slurm@pinnacles.ucmerced.edu #Pinnacle UCMerced Cluster + - no-reply@pace.gatech.edu #HIVE Slurm + - slurm@gkeyll-vc-test00.novalocal #VLab PlasmaScience JS2 cluster + - slurm@purdue.edu #Anvil Purdue + - SLURM User + - slurm@batch1.frontera.tacc.utexas.edu #Frontera + + - jobManagerType: UGE + emailParser: org.apache.airavata.monitor.email.parser.UGEEmailParser + resourceEmailAddresses: + - ls4.tacc.utexas.edu # contain Lonestar + - root # USD HPC Cluster + - root # SIU Little Dog + - sge@bigdog.research.siu.edu # SIU Big Dog + - root # USD HPC Cluster + + - jobManagerType: LSF + emailParser: org.apache.airavata.monitor.email.parser.LSFEmailParser + resourceEmailAddresses: + - iu.xsede.edu # test resource mail address + - tcs.tulsahpc.org #Tandy + + - jobManagerType: HTCONDOR + emailParser: org.apache.airavata.monitor.email.parser.HTCondorEmailParser + resourceEmailAddresses: + - condor@js-169-152.jetstream-cloud.org + - Owner of HTCondor Daemons #EHT Condor Access point + - Owner of HTCondor Daemons + - slurm@br003.ib.bridges2.psc.edu # AutoDock_Vina +# - jobManagerType: HTCondor +# emailParser: org.apache.airavata.monitor.email.parser.HTCondorEmailParser +# resourceEmailAddresses: + diff --git a/dev-tools/ansible/roles/airavata_services/templates/log4j2.xml.j2 b/dev-tools/ansible/roles/airavata_services/templates/log4j2.xml.j2 new file mode 100644 index 0000000000..3c8e581507 --- /dev/null +++ b/dev-tools/ansible/roles/airavata_services/templates/log4j2.xml.j2 @@ -0,0 +1,55 @@ + + + + + ${env:SERVICE_NAME:-output} + + + + + + + + %d [%t] %-5p %c{30} %X - %m%n + + + + + + + + + + + + + + + + + + + + + + diff --git a/dev-tools/ansible/roles/api-orch/defaults/main.yml b/dev-tools/ansible/roles/api-orch/defaults/main.yml index 0f6fa218b5..76ea2d6f1b 100644 --- a/dev-tools/ansible/roles/api-orch/defaults/main.yml +++ b/dev-tools/ansible/roles/api-orch/defaults/main.yml @@ -18,6 +18,9 @@ # under the License. # +api_orch_dir: "{{ deployment_dir | default(user_home + '/airavata') }}" +airavata_dist: "apache-airavata-{{ airavata_version | default('0.21-SNAPSHOT') }}" +airavata_dist_name: "{{ airavata_dist }}.tar.gz" api_orch_server_names: "api-orch" api_orch_log_dir: "{{ api_orch_dir }}/{{ airavata_dist }}/logs" api_orch_log_max_history: 30 @@ -49,11 +52,18 @@ thrift_client_pool_abandoned_removal_enabled: false thrift_client_pool_abandoned_removal_logged: false api_server_public_hostname: "localhost" +api_server_port: 8930 +api_server_tls_port: 9930 +api_server_host: "localhost" haproxy_api_server_ssl_cert: "/etc/ssl/{{ api_server_public_hostname }}/{{ api_server_public_hostname }}.pem" api_server_letsencrypt_ssl_cert: "/etc/letsencrypt/live/{{ api_server_public_hostname }}/cert.pem" haproxy_service_name: CentOS_7: haproxy18 Rocky_8: haproxy + Ubuntu_22: haproxy + Ubuntu_24: haproxy haproxy_config_dir: CentOS_7: /etc/haproxy18/ Rocky_8: /etc/haproxy/ + Ubuntu_22: /etc/haproxy/ + Ubuntu_24: /etc/haproxy/ diff --git a/dev-tools/ansible/roles/api-orch/files/prepareLetsEncryptCertificates.sh b/dev-tools/ansible/roles/api-orch/files/prepareLetsEncryptCertificates.sh index 81387518a5..ff7db5ed25 100644 --- a/dev-tools/ansible/roles/api-orch/files/prepareLetsEncryptCertificates.sh +++ b/dev-tools/ansible/roles/api-orch/files/prepareLetsEncryptCertificates.sh @@ -11,4 +11,9 @@ for CERTIFICATE in `find /etc/letsencrypt/live/* -type d`; do mkdir -p /etc/ssl/$CERTIFICATE/ cat /etc/letsencrypt/live/$CERTIFICATE/fullchain.pem /etc/letsencrypt/live/$CERTIFICATE/privkey.pem > /etc/ssl/$CERTIFICATE/$CERTIFICATE.pem + chmod 640 /etc/ssl/$CERTIFICATE/$CERTIFICATE.pem + chmod 750 /etc/ssl/$CERTIFICATE/ + chown root:haproxy /etc/ssl/$CERTIFICATE/$CERTIFICATE.pem + chown root:haproxy /etc/ssl/$CERTIFICATE/ + done diff --git a/dev-tools/ansible/roles/api-orch/tasks/haproxy/install_deps_Ubuntu_22.yml b/dev-tools/ansible/roles/api-orch/tasks/haproxy/install_deps_Ubuntu_22.yml new file mode 100644 index 0000000000..2ba64702ac --- /dev/null +++ b/dev-tools/ansible/roles/api-orch/tasks/haproxy/install_deps_Ubuntu_22.yml @@ -0,0 +1,30 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- + +- name: apt install haproxy (Ubuntu) + apt: + name: haproxy + state: present + update_cache: yes + become: true + become_user: root + diff --git a/dev-tools/ansible/roles/api-orch/tasks/main.yml b/dev-tools/ansible/roles/api-orch/tasks/main.yml index 3128c8a926..57adb00323 100644 --- a/dev-tools/ansible/roles/api-orch/tasks/main.yml +++ b/dev-tools/ansible/roles/api-orch/tasks/main.yml @@ -23,7 +23,6 @@ # api-orch deployment - name: Create api-orchestrator deployment directory file: path="{{ api_orch_dir }}" state=directory owner="{{ user }}" group="{{ group }}" - when: build is success - name: Check previous deployments stat: path="{{ api_orch_dir }}/{{ airavata_dist }}" get_md5=no get_checksum=no @@ -40,48 +39,6 @@ become: yes become_user: root -- name: Delete previous deployments - file: path="{{ api_orch_dir }}/{{ airavata_dist }}" state=absent - - -- name: Copy distribution to api-orcheatrator deployment directory - unarchive: src="{{ airavata_source_dir }}/modules/distribution/target/{{ airavata_dist_name }}" - dest="{{ api_orch_dir }}/" - copy=no - -- name: set api-orch private ip - set_fact: - api_server_host: "{{ api_server_bind_host }}" - orchestrator_host: "{{ orchestrator_bind_host }}" - cred_store_server_host: "{{ cred_store_server_bind_host }}" - registry_host: "{{ registry_bind_host }}" - sharing_registry_host: "{{ sharing_registry_bind_host }}" - profile_service_host: "{{ profile_service_bind_host }}" - - -- name: Copy Airavata server properties file - template: src=airavata-server.properties.j2 - dest="{{ api_orch_dir }}/{{ airavata_dist }}/bin/airavata-server.properties" - owner={{ user }} - group={{ group }} - mode="u=rw,g=r,o=r" - -- name: Copy logback configuration file - template: src=log4j2.xml.j2 - dest="{{ api_orch_dir }}/{{ airavata_dist }}/bin/log4j2.xml" - owner={{ user }} - group={{ group }} - mode="u=rw,g=r,o=r" - -- name: create logs directory - file: path="{{ api_orch_log_dir }}" state=directory owner={{ user }} group={{ group }} - -- name: Copy MariaDB connector jar to lib - get_url: url="{{ mariadb_connector_jar_url }}" - dest="{{ api_orch_dir }}/{{ airavata_dist }}/lib/" - owner={{ user }} - group={{ group }} - # Create a SSL certificate for the api server - name: allow http for Let's Encrypt certificate renewal @@ -107,12 +64,18 @@ become_user: root - name: generate certificate if it doesn't exist - command: certbot --standalone --non-interactive --agree-tos --email "{{ letsencrypt_email }}" -d {{ api_server_public_hostname }} certonly + command: certbot --apache --non-interactive --agree-tos --email "{{ letsencrypt_email }}" -d {{ api_server_public_hostname }} certonly become_user: root when: not stat_api_server_ssl_cert_result.stat.exists + register: certbot_result + +- name: run prepareLetsEncryptCertificates.sh script to combine certificates + shell: "{{ haproxy_config_dir[ansible_distribution + '_' + ansible_distribution_major_version]}}/prepareLetsEncryptCertificates.sh" + become_user: root + when: stat_api_server_ssl_cert_result.stat.exists or (certbot_result is defined and certbot_result is succeeded) - name: set certificate renewal post-hook - command: certbot renew --force-renewal --installer null --standalone --post-hook "{{ haproxy_config_dir[ansible_distribution + '_' + ansible_distribution_major_version]}}/prepareLetsEncryptCertificates.sh && systemctl reload {{ haproxy_service_name[ansible_distribution + '_' + ansible_distribution_major_version]}}.service" --quiet + command: certbot renew --force-renewal --installer null --webroot --post-hook "{{ haproxy_config_dir[ansible_distribution + '_' + ansible_distribution_major_version]}}/prepareLetsEncryptCertificates.sh && systemctl reload {{ haproxy_service_name[ansible_distribution + '_' + ansible_distribution_major_version]}}.service" --quiet become_user: root # Renewal might fail due to rate limiting, which is fine since we only need to set the post-hook ignore_errors: true @@ -128,6 +91,16 @@ - name: Install HAProxy include_tasks: haproxy/install_deps_{{ ansible_distribution }}_{{ ansible_distribution_major_version }}.yml + when: ansible_os_family == "RedHat" + +- name: Install HAProxy (Ubuntu) + apt: + name: haproxy + state: present + update_cache: yes + become: true + become_user: root + when: ansible_os_family == "Debian" - name: Copy HAProxy config file template: src=haproxy.cfg.j2 @@ -149,8 +122,7 @@ state: enabled immediate: yes rich_rule: rule family=ipv4 source address="{{ item }}" port port="{{ sharing_registry_port }}" protocol=tcp accept - with_items: - - "{{ sharing_subnets }}" + with_items: "{{ sharing_subnets | default(['0.0.0.0/0']) }}" become_user: root - name: allow only selected networks to access Airavata Registry @@ -160,8 +132,7 @@ state: enabled immediate: yes rich_rule: rule family=ipv4 source address="{{ item }}" port port="{{ registry_port }}" protocol=tcp accept - with_items: - - "{{ registry_subnets }}" + with_items: "{{ registry_subnets | default(['0.0.0.0/0']) }}" become_user: root - name: allow only selected networks to access Airavata Credential Store @@ -171,8 +142,7 @@ state: enabled immediate: yes rich_rule: rule family=ipv4 source address="{{ item }}" port port="{{ cred_store_port }}" protocol=tcp accept - with_items: - - "{{ credential_store_subnets }}" + with_items: "{{ credential_store_subnets | default(['0.0.0.0/0']) }}" become_user: root - name: allow all networks to access Airavata API Server over TLS @@ -200,34 +170,7 @@ state: enabled immediate: yes rich_rule: rule family=ipv4 source address="{{ item }}" port port="{{ api_server_monitoring_port }}" protocol=tcp accept - with_items: - - "{{ monitoring_subnets }}" - become_user: root - -- name: Install api-orch systemd script - template: src=apiorch.service.j2 - dest="{{ api_orch_systemd_unit_file }}" - become: yes - become_user: root - - -- name: Allow to modify files - sefcontext: - target: "{{ api_orch_dir }}/{{ airavata_dist }}/bin/airavata-server-start.sh" - setype: initrc_exec_t - state: present - become: yes + with_items: "{{ monitoring_subnets | default(['0.0.0.0/0']) }}" become_user: root - when: ansible_distribution == "Rocky" -- name: Apply new SELinux file context to filesystem - command: restorecon -v "{{ api_orch_dir }}/{{ airavata_dist }}/bin/airavata-server-start.sh" - become: yes - become_user: root - when: ansible_distribution == "Rocky" - -- name: start api-orch - service: name=apiorch state=started enabled=yes daemon_reload=yes - become: yes - become_user: root ... diff --git a/dev-tools/ansible/roles/api-orch/templates/airavata-server.properties.j2 b/dev-tools/ansible/roles/api-orch/templates/airavata-server.properties.j2 index 18680aabd7..c6aa47d39a 100644 --- a/dev-tools/ansible/roles/api-orch/templates/airavata-server.properties.j2 +++ b/dev-tools/ansible/roles/api-orch/templates/airavata-server.properties.j2 @@ -105,7 +105,7 @@ sharingcatalog.validationQuery=SELECT 1 from CONFIGURATION # Sharing Registry Server Configuration ########################################################################### sharing_server=org.apache.airavata.sharing.registry.server.SharingRegistryServer -sharing.registry.server.host={{ sharing_registry_host }} +sharing.registry.server.host={{ sharing_registry_bind_host }} sharing.registry.server.port={{ sharing_registry_port }} ########################################################################### @@ -120,14 +120,14 @@ sharing.registry.server.port={{ sharing_registry_port }} # API Server Configurations ########################################################################### apiserver.class=org.apache.airavata.api.server.AiravataAPIServer -apiserver.host={{ api_server_host }} +apiserver.host={{ api_server_bind_host }} apiserver.port={{ api_server_port }} ########################################################################### # Orchestrator Server Configurations ########################################################################### orchestrator=org.apache.airavata.orchestrator.server.OrchestratorServer -orchestrator.server.host={{ orchestrator_host }} +orchestrator.server.host={{ orchestrator_bind_host }} orchestrator.server.port={{ orchestrator_port }} orchestrator.server.min.threads=50 job.validators=org.apache.airavata.orchestrator.core.validator.impl.BatchQueueValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator @@ -138,7 +138,7 @@ host.scheduler=org.apache.airavata.orchestrator.core.schedule.DefaultHostSchedul # Registry Server Configurations ########################################################################### regserver=org.apache.airavata.registry.api.service.RegistryAPIServer -regserver.server.host={{registry_host}} +regserver.server.host={{registry_bind_host}} regserver.server.port={{registry_port}} regserver.server.min.threads=50 @@ -163,7 +163,7 @@ credential.store.jdbc.url=jdbc:mariadb://{{ db_server }}:3306/{{ credential_stor credential.store.jdbc.user={{ db_user }} credential.store.jdbc.password={{ db_password }} credential.store.jdbc.driver=org.mariadb.jdbc.Driver -credential.store.server.host={{ cred_store_server_host }} +credential.store.server.host={{ cred_store_server_bind_host }} credential.store.server.port={{ cred_store_port }} credential.store.class=org.apache.airavata.credential.store.server.CredentialStoreServer credential.store.jdbc.validationQuery=SELECT 1 from CONFIGURATION @@ -245,7 +245,7 @@ in.memory.cache.size=1000 ########################################################################### # Profile Service Configuration ########################################################################### -profile.service.server.host={{ profile_service_host }} +profile.service.server.host={{ profile_service_bind_host }} profile.service.server.port={{ profile_service_port }} profile_service.class=org.apache.airavata.service.profile.server.ProfileServiceServer # MariaDB properties diff --git a/dev-tools/ansible/roles/api-orch/templates/haproxy.cfg.j2 b/dev-tools/ansible/roles/api-orch/templates/haproxy.cfg.j2 index d7352896ae..e31bd14e1e 100644 --- a/dev-tools/ansible/roles/api-orch/templates/haproxy.cfg.j2 +++ b/dev-tools/ansible/roles/api-orch/templates/haproxy.cfg.j2 @@ -45,11 +45,12 @@ global #--------------------------------------------------------------------- defaults log global - option httplog + mode tcp + option tcplog option dontlognull - option http-server-close - option forwardfor except 127.0.0.0/8 - option redispatch + timeout connect 5s + timeout client 50s + timeout server 50s #--------------------------------------------------------------------- # main frontend which proxys to the backends @@ -61,10 +62,6 @@ frontend main bind *:{{ api_server_tls_port }} ssl crt {{ haproxy_api_server_ssl_cert }} default_backend fix-backend -#--------------------------------------------------------------------- -# static backend for serving up images, stylesheets and such -#--------------------------------------------------------------------- - #--------------------------------------------------------------------- # round robin balancing between the various backends #--------------------------------------------------------------------- @@ -72,4 +69,7 @@ backend fix-backend mode tcp log global option tcplog - server quickfix {{ api_server_host }}:{{ api_server_port }} check + timeout connect 5s + timeout client 50s + timeout server 50s + server quickfix 127.0.0.1:{{ api_server_port }} check diff --git a/dev-tools/ansible/roles/common/defaults/main.yml b/dev-tools/ansible/roles/common/defaults/main.yml index c5f496d690..a6919e832b 100644 --- a/dev-tools/ansible/roles/common/defaults/main.yml +++ b/dev-tools/ansible/roles/common/defaults/main.yml @@ -21,5 +21,5 @@ keystore_src_path: "airavata.p12" cred_keystore_src_path: "airavata.p12" -apache_maven_version: "apache-maven-3.6.3" -apache_maven_url: "https://www-eu.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz" +apache_maven_version: "apache-maven-3.9.11" +apache_maven_url: "https://dlcdn.apache.org/maven/maven-3/3.9.11/binaries/apache-maven-3.9.11-bin.tar.gz" diff --git a/dev-tools/ansible/roles/common/tasks/main.yml b/dev-tools/ansible/roles/common/tasks/main.yml index 58a2b63b63..b7b2db723d 100644 --- a/dev-tools/ansible/roles/common/tasks/main.yml +++ b/dev-tools/ansible/roles/common/tasks/main.yml @@ -33,10 +33,42 @@ become_user: root when: ansible_distribution == "Rocky" +- name: Install Airavata pre-requireties (Ubuntu/Debian) + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - git + - build-essential + - automake + - bison + - flex + - libboost-all-dev + - libevent-dev + - libssl-dev + - libtool + - pkg-config + become_user: root + when: ansible_os_family == "Debian" + +- name: Check if Maven archive already exists + stat: + path: "/opt/{{ apache_maven_version }}-bin.tar.gz" + register: maven_archive + become: yes + - name: download {{ apache_maven_version }} - get_url: url="{{ apache_maven_url }}" dest="/opt/{{ apache_maven_version }}-bin.tar.gz" + shell: | + cd /tmp && \ + (wget -q --no-check-certificate "{{ apache_maven_url }}" -O "{{ apache_maven_version }}-bin.tar.gz" || \ + curl -k -L "{{ apache_maven_url }}" -o "{{ apache_maven_version }}-bin.tar.gz") && \ + mv "/tmp/{{ apache_maven_version }}-bin.tar.gz" "/opt/{{ apache_maven_version }}-bin.tar.gz" + args: + creates: "/opt/{{ apache_maven_version }}-bin.tar.gz" become: yes become_user: root + when: not maven_archive.stat.exists - name: unzip maven unarchive: @@ -53,6 +85,45 @@ become: yes become_user: root +- name: Check if Thrift is installed + command: which thrift + register: thrift_check + changed_when: false + failed_when: false + become: no + +- name: Download Thrift 0.22.0 + shell: | + cd /tmp && \ + wget -q --no-check-certificate https://dlcdn.apache.org/thrift/0.22.0/thrift-0.22.0.tar.gz && \ + tar -xzf thrift-0.22.0.tar.gz + args: + creates: /tmp/thrift-0.22.0 + become: yes + become_user: root + when: thrift_check.rc != 0 + +- name: Build and install Thrift 0.22.0 + shell: | + cd /tmp/thrift-0.22.0 && \ + ./configure --without-rs --enable-libs=no --enable-tests=no && \ + make -j$(nproc) && \ + make install && \ + ldconfig + become: yes + become_user: root + when: thrift_check.rc != 0 + +- name: Verify Thrift installation + command: thrift -version + register: thrift_version + changed_when: false + become: no + +- name: Display Thrift version + debug: + msg: "Thrift version: {{ thrift_version.stdout }}" + # Setup airavata source - name: Create deployment directory {{ deployment_dir }} file: path={{ deployment_dir }} state=directory mode=0755 @@ -64,8 +135,8 @@ owner={{ user }} group={{ group }} -- name: git checkout from airavata github repo {{ airavata_repo }} branch {{ git_branch }} - git: repo="{{ airavata_repo }}" +- name: git checkout from airavata github repo {{ airavata_git_repo }} branch {{ git_branch }} + git: repo="{{ airavata_git_repo }}" dest="{{ airavata_source_dir }}" version="{{ git_branch }}" register: checkout @@ -75,29 +146,8 @@ command: /opt/{{apache_maven_version}}/bin/mvn clean install -Dmaven.test.skip=true chdir="{{ airavata_source_dir }}/" environment: MAVEN_OPTS: "-Xmx2048m" + PATH: "/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin:{{ ansible_env.PATH }}" register: build tags: update # when: (checkout|success) and (checkout.changed == true) -################################################################################ -# copy key store and trust store files -- name: Create KeyStores directory - file: path={{ keystores_location }} - state=directory - owner={{ user }} group={{ group }} - -- name: Transfer airavata.p12 KeyStore file - copy: src={{ keystore_src_path }} - dest="{{ keystores_location }}/{{ keystore_src_path | basename }}" - owner={{ user }} group={{ group }} - -- name: Transfer airavata.p12 KeyStore file - copy: src={{ cred_keystore_src_path }} - dest="{{ keystores_location }}/{{ cred_keystore_src_path | basename }}" - owner={{ user }} group={{ group }} - -- name: Transfer client trust store KeyStore file - copy: src={{ client_truststore_src_path }} - dest="{{ keystores_location }}/{{ client_truststore_src_path | basename }}" - owner={{ user }} group={{ group }} - when: client_truststore_src_path is defined diff --git a/dev-tools/ansible/roles/database/tasks/keycloak.yml b/dev-tools/ansible/roles/database/tasks/keycloak.yml index c8f10bf276..b6461e7eef 100644 --- a/dev-tools/ansible/roles/database/tasks/keycloak.yml +++ b/dev-tools/ansible/roles/database/tasks/keycloak.yml @@ -22,20 +22,47 @@ # Setup keycloak user and database - name: create keycloak database - mysql_db: name="keycloak" state=present encoding=utf8 + mysql_db: + name: "keycloak" + state: present + encoding: utf8 + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: "{{ '/var/run/mysqld/mysqld.sock' if ansible_os_family == 'Debian' else '/var/lib/mysql/mysql.sock' }}" when: "'keycloak' in groups" - name: give access to {{ keycloak_db_username }} from remote # Creating the access record with the IP address works better for MySQL so it # doesn't have to do a DNS lookup (and it has DNS caching issues if the domain # name changes) - mysql_user: name="{{ keycloak_db_username }}" password="{{ keycloak_db_password }}" host="{{ hostvars[item].public_ipv4 | default(item) }}" + mysql_user: + name: "{{ keycloak_db_username }}" + password: "{{ keycloak_db_password }}" + host: "{{ hostvars[item].public_ipv4 | default(hostvars[item].ansible_host | default(item)) }}" + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: "{{ '/var/run/mysqld/mysqld.sock' if ansible_os_family == 'Debian' else '/var/lib/mysql/mysql.sock' }}" with_items: - "{{ groups['keycloak'] }}" + when: "'keycloak' in groups and (hostvars[item].public_ipv4 is defined or hostvars[item].ansible_host is defined)" + +- name: give access to {{ keycloak_db_username }} from localhost + mysql_user: + name: "{{ keycloak_db_username }}" + password: "{{ keycloak_db_password }}" + host: localhost + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: "{{ '/var/run/mysqld/mysqld.sock' if ansible_os_family == 'Debian' else '/var/lib/mysql/mysql.sock' }}" - name: create new user {{ keycloak_db_username }} with all privilege - mysql_user: name="{{ keycloak_db_username }}" - password="{{ keycloak_db_password }}" - append_privs=yes - host_all=yes - priv=keycloak.*:ALL,GRANT state=present + mysql_user: + name: "{{ keycloak_db_username }}" + password: "{{ keycloak_db_password }}" + append_privs: yes + host_all: yes + priv: keycloak.*:ALL,GRANT + state: present + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: "{{ '/var/run/mysqld/mysqld.sock' if ansible_os_family == 'Debian' else '/var/lib/mysql/mysql.sock' }}" diff --git a/dev-tools/ansible/roles/database/tasks/main.yml b/dev-tools/ansible/roles/database/tasks/main.yml index e34a716fa6..19acf0f2de 100644 --- a/dev-tools/ansible/roles/database/tasks/main.yml +++ b/dev-tools/ansible/roles/database/tasks/main.yml @@ -39,13 +39,28 @@ when: ansible_distribution == "CentOS" - name: install pip (Rocky) - dnf: name=python39-pip + dnf: name=python3-pip become_user: root when: ansible_distribution == "Rocky" -- name: install pexpect - pip: name=pexpect +- name: install pexpect (Ubuntu/Debian) + apt: + name: python3-pexpect + state: present + update_cache: yes become_user: root + when: ansible_os_family == "Debian" + +- name: install pexpect (Rocky) + dnf: name=python3-pexpect + become_user: root + when: ansible_distribution == "Rocky" + +- name: install pexpect (CentOS) + pip: + name: pexpect + become_user: root + when: ansible_distribution == "CentOS" # - name: Adds Python MySQL support on Debian/Ubuntu # apt: pkg="python-mysqldb" state=present @@ -57,10 +72,18 @@ when: ansible_distribution == 'CentOS' - name: Adds Python MySQL support on Rocky - dnf: name=python3-mysql state=present + dnf: name=python3-PyMySQL state=present become_user: root when: ansible_distribution == 'Rocky' +- name: Adds Python MySQL support on Ubuntu/Debian + apt: + pkg: python3-pymysql + state: present + update_cache: yes + become_user: root + when: ansible_os_family == 'Debian' + - name: Add MariaDB yum repository on CentOS {{ ansible_distribution }} copy: src="MariaDB_yum_CentOS_{{ ansible_distribution_major_version }}.repo" dest="/etc/yum.repos.d/" @@ -79,6 +102,33 @@ become_user: root when: ansible_distribution == 'Rocky' +- name: Set MariaDB root password via debconf (Ubuntu/Debian) + debconf: + name: mariadb-server + question: "mysql-server/root_password" + value: "{{ mysql_root_password | default('') }}" + vtype: password + become_user: root + when: ansible_os_family == 'Debian' and mysql_root_password is defined and mysql_root_password | length > 0 + +- name: Confirm MariaDB root password via debconf (Ubuntu/Debian) + debconf: + name: mariadb-server + question: "mysql-server/root_password_again" + value: "{{ mysql_root_password | default('') }}" + vtype: password + become_user: root + when: ansible_os_family == 'Debian' and mysql_root_password is defined and mysql_root_password | length > 0 + +- name: install mariadb (Ubuntu/Debian) + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: "{{ mysql_packages }}" + become_user: root + when: ansible_os_family == 'Debian' + - name: check if mysql has been updated stat: path=/usr/share/mysql/SELinux/mariadb.pp register: mysql_selinux_update @@ -88,25 +138,25 @@ - name: double check policycoreutils installed (Centos) yum: name=policycoreutils-python state=installed - when: mysql_selinux_update.stat.exists == False and (ansible_distribution == 'CentOS') + when: mysql_selinux_update.stat.exists == False and (ansible_distribution == 'CentOS') and ansible_os_family == "RedHat" become_user: root - name: double check policycoreutils installed (Rocky) dnf: name=policycoreutils-python-utils state=installed - when: mysql_selinux_update.stat.exists == False and ansible_distribution == 'Rocky' + when: mysql_selinux_update.stat.exists == False and ansible_distribution == 'Rocky' and ansible_os_family == "RedHat" become_user: root - name: Copy SELinux type enforcement file copy: src=mysql-tmp.te dest=/tmp/ - when: mysql_selinux_update.stat.exists == False + when: mysql_selinux_update.stat.exists == False and ansible_os_family == "RedHat" - name: Compile SELinux module file command: checkmodule -M -m -o /tmp/mysql-tmp.mod /tmp/mysql-tmp.te - when: mysql_selinux_update.stat.exists == False + when: mysql_selinux_update.stat.exists == False and ansible_os_family == "RedHat" - name: Build SELinux policy package command: semodule_package -o /tmp/mysql-tmp.pp -m /tmp/mysql-tmp.mod - when: mysql_selinux_update.stat.exists == False + when: mysql_selinux_update.stat.exists == False and ansible_os_family == "RedHat" #- name: check if mysql has been updated # stat: path=/tmp/mysql-tmp.pp @@ -117,23 +167,50 @@ - name: unLoad SELinux policy package command: semodule -r mysql-tmp - when: mysql_selinux_update.stat.exists == False + when: mysql_selinux_update.stat.exists == False and ansible_os_family == "RedHat" become_user: root ignore_errors: True - name: Load SELinux policy package command: semodule -i /tmp/mysql-tmp.pp - when: mysql_selinux_update.stat.exists == False + when: mysql_selinux_update.stat.exists == False and ansible_os_family == "RedHat" become_user: root - name: Remove temporary files file: path=/tmp/mysql-tmp.* state=absent - when: mysql_selinux_update.stat.exists == False + when: mysql_selinux_update.stat.exists == False and ansible_os_family == "RedHat" # TODO: SELinux issue for 10.0.29 MariaDB, need to allow setgid/setuid. ## See https://www.rootusers.com/how-to-fix-mariadb-10-0-29-selinux-update-failure/ # and https://jira.mariadb.org/browse/MDEV-11789 # For now I'm not adding to Ansible script since this bug should be fixed in next MariaDB release + +- name: Stop stray MariaDB processes (CentOS/Rocky) + shell: pkill -9 -f "mariadbd|mysqld_safe|mysqld" || true + become_user: root + when: ansible_os_family == 'RedHat' + ignore_errors: yes + +- name: Remove stale MariaDB socket and pid files (CentOS/Rocky) + file: + path: "{{ item }}" + state: absent + with_items: + - /var/lib/mysql/mysql.sock + - /var/run/mariadb/mariadb.pid + - /run/mariadb/mariadb.pid + become_user: root + when: ansible_os_family == 'RedHat' + ignore_errors: yes + +- name: Ensure no MariaDB systemd overrides remain from recovery attempts (CentOS/Rocky) + file: + path: /etc/systemd/system/mariadb.service.d + state: absent + become_user: root + when: ansible_os_family == 'RedHat' + ignore_errors: yes + - name: start mariadb (CentOS) service: name=mysql state=started enabled=yes become_user: root @@ -144,34 +221,157 @@ become_user: root when: ansible_distribution == 'Rocky' +- name: Wait for MariaDB to be ready (CentOS/Rocky) + wait_for: + port: 3306 + host: localhost + delay: 5 + timeout: 60 + become_user: root + when: ansible_os_family == 'RedHat' + +- name: start mariadb (Ubuntu/Debian) + service: name=mariadb state=started enabled=yes + become_user: root + when: ansible_os_family == 'Debian' + +- name: Wait for MariaDB to be ready + wait_for: + port: 3306 + host: localhost + delay: 5 + timeout: 60 + become_user: root + when: ansible_os_family == 'Debian' + +- name: Wait for MariaDB socket to be ready (Ubuntu/Debian) + wait_for: + path: /var/run/mysqld/mysqld.sock + state: present + timeout: 60 + become_user: root + when: ansible_os_family == 'Debian' + - include: secure_install.yml - name: create databases - mysql_db: name="{{ item }}" state=present encoding=utf8 + mysql_db: + name: "{{ item }}" + state: present + encoding: utf8 + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: "/var/run/mysqld/mysqld.sock" with_items: - "{{ mysql_databases }}" + when: ansible_os_family == 'Debian' + +- name: create databases (CentOS/Rocky) + mysql_db: + name: "{{ item }}" + state: present + encoding: utf8 + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: /var/lib/mysql/mysql.sock + with_items: + - "{{ mysql_databases }}" + when: ansible_os_family == 'RedHat' - name: give access to {{ db_user }} from remote - mysql_user: name="{{ db_user }}" password="{{ db_password }}" host="{{ hostvars[item]['ansible_default_ipv4']['address'] }}" + mysql_user: + name: "{{ db_user }}" + password: "{{ db_password }}" + host: "{{ hostvars[item].ansible_default_ipv4.address | default(hostvars[item].ansible_host | default(item)) }}" + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: "/var/run/mysqld/mysqld.sock" with_items: - "{{ groups['api-orch'] }}" + when: ansible_os_family == 'Debian' and 'api-orch' in groups and (hostvars[item].ansible_default_ipv4.address is defined or hostvars[item].ansible_host is defined) + +- name: give access to {{ db_user }} from remote (CentOS/Rocky) + mysql_user: + name: "{{ db_user }}" + password: "{{ db_password }}" + host: "{{ hostvars[item].ansible_default_ipv4.address | default(hostvars[item].ansible_host | default(item)) }}" + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: /var/lib/mysql/mysql.sock + with_items: + - "{{ groups['api-orch'] }}" + when: ansible_os_family == 'RedHat' and 'api-orch' in groups and (hostvars[item].ansible_default_ipv4.address is defined or hostvars[item].ansible_host is defined) - name: give access to {{ db_user }} from localhost - mysql_user: name="{{ db_user }}" password="{{ db_password }}" host="localhost" + mysql_user: + name: "{{ db_user }}" + password: "{{ db_password }}" + host: localhost + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: "/var/run/mysqld/mysqld.sock" + when: ansible_os_family == 'Debian' + +- name: give access to {{ db_user }} from localhost (CentOS/Rocky) + mysql_user: + name: "{{ db_user }}" + password: "{{ db_password }}" + host: localhost + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: /var/lib/mysql/mysql.sock + when: ansible_os_family == 'RedHat' # TODO: database access from GFac is no longer needed (GFac deprecated and only using Registry API) - name: give access to {{ db_user }} from remote - mysql_user: name="{{ db_user }}" password="{{ db_password }}" host="{{ hostvars[item]['ansible_default_ipv4']['address'] }}" + mysql_user: + name: "{{ db_user }}" + password: "{{ db_password }}" + host: "{{ hostvars[item].ansible_default_ipv4.address | default(hostvars[item].ansible_host | default(item)) }}" + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: "/var/run/mysqld/mysqld.sock" + with_items: + - "{{ groups['gfac'] }}" + when: ansible_os_family == 'Debian' and 'gfac' in groups and (hostvars[item].ansible_default_ipv4.address is defined or hostvars[item].ansible_host is defined) + +- name: give access to {{ db_user }} from remote (CentOS/Rocky) + mysql_user: + name: "{{ db_user }}" + password: "{{ db_password }}" + host: "{{ hostvars[item].ansible_default_ipv4.address | default(hostvars[item].ansible_host | default(item)) }}" + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: /var/lib/mysql/mysql.sock with_items: - "{{ groups['gfac'] }}" - when: "'gfac' in groups" + when: ansible_os_family == 'RedHat' and 'gfac' in groups and (hostvars[item].ansible_default_ipv4.address is defined or hostvars[item].ansible_host is defined) - name: create new user {{ db_user }} with all privilege - mysql_user: name="{{ db_user }}" - password="{{ db_password }}" - append_privs=yes - host_all=yes - priv=*.*:ALL,GRANT state=present + mysql_user: + name: "{{ db_user }}" + password: "{{ db_password }}" + append_privs: yes + host_all: yes + priv: "*.*:ALL,GRANT" + state: present + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: "/var/run/mysqld/mysqld.sock" + when: ansible_os_family == 'Debian' + +- name: create new user {{ db_user }} with all privilege (CentOS/Rocky) + mysql_user: + name: "{{ db_user }}" + password: "{{ db_password }}" + append_privs: yes + host_all: yes + priv: "*.*:ALL,GRANT" + state: present + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: /var/lib/mysql/mysql.sock + when: ansible_os_family == 'RedHat' - include: keycloak.yml when: "'keycloak' in groups" @@ -182,7 +382,7 @@ permanent: yes state: enabled immediate: yes - rich_rule: rule family=ipv4 source address="{{ item }}" port port="{{ db_server_port }}" protocol=tcp accept - with_items: - - "{{ db_subnets }}" + rich_rule: rule family=ipv4 source address="{{ item }}" port port="{{ db_server_port | default(3306) }}" protocol=tcp accept + with_items: "{{ db_subnets | default([]) }}" become_user: root + when: db_subnets is defined and db_subnets | length > 0 diff --git a/dev-tools/ansible/roles/database/tasks/secure_install.yml b/dev-tools/ansible/roles/database/tasks/secure_install.yml index 27033de170..0b360b36cc 100644 --- a/dev-tools/ansible/roles/database/tasks/secure_install.yml +++ b/dev-tools/ansible/roles/database/tasks/secure_install.yml @@ -20,24 +20,195 @@ --- # This is ansible equivalent for mysql_secure_installation -- name: Sets the root password - mysql_user: user=root - password="{{ mysql_root_password }}" - host=localhost - login_user=root - # login_password="{{ mysql_root_password }}" +- name: Skip root password setting for Ubuntu - handled by debconf or already configured + debug: + msg: "Root password is set via debconf during installation (fresh installs) or already configured. Skipping password setting step." + when: ansible_os_family == 'Debian' + +- name: Mark root password as set (Ubuntu/Debian) + file: + path: /tmp/.root_password_set + state: touch + become: yes + when: ansible_os_family == 'Debian' + +- name: Reset and set root password via recovery mode (CentOS/Rocky) + block: + - name: Stop MariaDB before recovery (CentOS/Rocky) + systemd: + name: mariadb + state: stopped + become: yes + become_user: root + + - name: Create systemd override directory for MariaDB recovery (CentOS/Rocky) + file: + path: /etc/systemd/system/mariadb.service.d + state: directory + become: yes + become_user: root + + - name: Configure MariaDB to start with skip-grant-tables (CentOS/Rocky) + copy: + dest: /etc/systemd/system/mariadb.service.d/override.conf + content: | + [Service] + ExecStart= + ExecStart=/usr/libexec/mariadbd --basedir=/usr $MYSQLD_OPTS $_WSREP_NEW_CLUSTER --skip-grant-tables --skip-networking --pid-file=/run/mariadb/mariadb.pid --socket=/var/lib/mysql/mysql.sock + become: yes + become_user: root + + - name: Start MariaDB in recovery mode (CentOS/Rocky) + systemd: + name: mariadb + daemon_reload: yes + state: started + become: yes + become_user: root + + - name: Wait for MariaDB socket in recovery mode (CentOS/Rocky) + wait_for: + path: /var/lib/mysql/mysql.sock + state: present + timeout: 60 + become: yes + become_user: root + + - name: Reset root password in recovery mode (CentOS/Rocky) + shell: | + mysql --protocol=socket --socket=/var/lib/mysql/mysql.sock -uroot <<'EOF' + FLUSH PRIVILEGES; + ALTER USER 'root'@'localhost' IDENTIFIED BY '{{ mysql_root_password }}'; + UPDATE mysql.user SET plugin='mysql_native_password' WHERE user='root' AND host='localhost'; + FLUSH PRIVILEGES; + EOF + become: yes + become_user: root + no_log: true + failed_when: false + rescue: + - debug: + msg: "Recovery mode password reset failed; attempting cleanup and restart." + always: + - name: Stop MariaDB recovery instance (CentOS/Rocky) + systemd: + name: mariadb + state: stopped + become: yes + become_user: root + ignore_errors: yes + + - name: Remove recovery override for MariaDB (CentOS/Rocky) + file: + path: /etc/systemd/system/mariadb.service.d/override.conf + state: absent + become: yes + become_user: root + ignore_errors: yes + + - name: Remove recovery override directory if empty (CentOS/Rocky) + file: + path: /etc/systemd/system/mariadb.service.d + state: absent + become: yes + become_user: root + ignore_errors: yes + + - name: Reload systemd and start MariaDB normally after recovery (CentOS/Rocky) + systemd: + name: mariadb + daemon_reload: yes + state: started + become: yes + become_user: root + + - name: Verify root login with provided password (CentOS/Rocky) + command: "mysql --protocol=socket --socket=/var/lib/mysql/mysql.sock -uroot -p{{ mysql_root_password }} -e 'SELECT 1'" + register: root_login_check_recovered + changed_when: false + failed_when: root_login_check_recovered.rc != 0 + no_log: true + become: yes + become_user: root + - name: Wait for MariaDB to be ready after recovery (CentOS/Rocky) + wait_for: + port: 3306 + host: localhost + delay: 2 + timeout: 60 + become: yes + become_user: root + when: ansible_os_family == 'RedHat' - name: Copy .my.cnf file template: src=my.cnf.j2 dest="{{ user_home }}/.my.cnf" # become: yes -- name: Removes all anonymous user accounts - mysql_user: name='' host_all=yes state=absent +- name: Removes all anonymous user accounts (Ubuntu/Debian) + mysql_user: + name: '' + host_all: yes + state: absent + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: "/var/run/mysqld/mysqld.sock" + when: ansible_os_family == 'Debian' + ignore_errors: yes + +- name: Removes all anonymous user accounts (CentOS/Rocky) + mysql_user: + name: '' + host_all: yes + state: absent + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: /var/lib/mysql/mysql.sock + when: ansible_os_family == 'RedHat' + become: yes + become_user: root + +- name: Secures the MySQL root user for all hosts (Ubuntu/Debian) + mysql_user: + user: root + password: "{{ mysql_root_password }}" + host_all: yes + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: "/var/run/mysqld/mysqld.sock" + when: ansible_os_family == 'Debian' + ignore_errors: yes + +- name: Secures the MySQL root user for all hosts (CentOS/Rocky) + mysql_user: + user: root + password: "{{ mysql_root_password }}" + host_all: yes + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: /var/lib/mysql/mysql.sock + when: ansible_os_family == 'RedHat' + become: yes + become_user: root -- name: Secures the MySQL root user for all hosts - mysql_user: user=root password="{{ mysql_root_password }}" host_all=yes +- name: Removes the MySQL test database (Ubuntu/Debian) + mysql_db: + db: test + state: absent + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: "/var/run/mysqld/mysqld.sock" + when: ansible_os_family == 'Debian' + ignore_errors: yes -- name: Removes the MySQL test database - mysql_db: db=test state=absent +- name: Removes the MySQL test database (CentOS/Rocky) + mysql_db: + db: test + state: absent + login_user: root + login_password: "{{ mysql_root_password }}" + login_unix_socket: /var/lib/mysql/mysql.sock + when: ansible_os_family == 'RedHat' + become: yes + become_user: root ... diff --git a/dev-tools/ansible/roles/database/vars/main.yml b/dev-tools/ansible/roles/database/vars/main.yml index 5bde0440c1..7f6bb2573d 100644 --- a/dev-tools/ansible/roles/database/vars/main.yml +++ b/dev-tools/ansible/roles/database/vars/main.yml @@ -30,6 +30,7 @@ mysql_databases: - "{{ credential_store }}" - "{{ sharing_catalog }}" - "{{ profile_service }}" + - "{{ research_catalog }}" mysql_privs: - "{{ app_catalog }}.*:ALL" @@ -39,5 +40,6 @@ mysql_privs: - "{{ credential_store }}.*:ALL" - "{{ sharing_catalog }}.*:ALL" - "{{ profile_service }}.*:ALL" + - "{{ research_catalog }}.*:ALL" ... diff --git a/dev-tools/ansible/roles/env_setup/tasks/main.yml b/dev-tools/ansible/roles/env_setup/tasks/main.yml index 31cbf4a703..fcce6eafd7 100644 --- a/dev-tools/ansible/roles/env_setup/tasks/main.yml +++ b/dev-tools/ansible/roles/env_setup/tasks/main.yml @@ -26,7 +26,11 @@ become: yes - name: Create a new user "{{ user }}" - user: name={{ user }} group={{ group }} + user: + name: "{{ user }}" + group: "{{ group }}" + shell: /bin/bash + create_home: yes become: yes - name: Install Firewalld (RedHat) @@ -38,7 +42,7 @@ apt: name=firewalld state=latest update_cache=yes become: yes when: ansible_os_family == "Debian" - + register: firewalld_install_debian # TODO: stop iptables service, can't have both iptables and firewalld on same host # firewalld is just a frontend for iptables - so we can't remove it @@ -49,9 +53,23 @@ # - iptables # - ip6tables +- name: Check if firewalld service file exists (systemd custom location) + stat: + path: /etc/systemd/system/firewalld.service + register: firewalld_service_file_systemd + become: yes + +- name: Check if firewalld service file exists (Debian/Ubuntu standard location) + stat: + path: /lib/systemd/system/firewalld.service + register: firewalld_service_file_lib + become: yes + when: ansible_os_family == "Debian" + - name: Start firewalld service - service: name=firewalld state=started + service: name=firewalld state=started enabled=yes become: yes + when: (firewalld_service_file_systemd.stat.exists | default(false)) or (firewalld_service_file_lib.stat.exists | default(false)) or ansible_os_family == "RedHat" - name: open firewall port 22 for SSH connections (Redhat or Rocky) firewalld: port="22/tcp" diff --git a/dev-tools/ansible/roles/java/tasks/main.yml b/dev-tools/ansible/roles/java/tasks/main.yml index e41caa1759..34c10f9819 100644 --- a/dev-tools/ansible/roles/java/tasks/main.yml +++ b/dev-tools/ansible/roles/java/tasks/main.yml @@ -19,23 +19,59 @@ # --- -- name: Install OpenJDK 11 (CentOS) - yum: name="{{ openjdk_version }}" state=present update_cache=yes +- name: Install OpenJDK 17 (CentOS) + yum: + name: "{{ openjdk_version }}" + state: present + update_cache: yes become: yes tags: - always when: ansible_distribution == "CentOS" -- name: Install OpenJDK 11 (Rocky) +- name: Install OpenJDK 17 (Rocky) dnf: name="{{ openjdk_version_rocky }}" become: yes tags: - always when: ansible_distribution == "Rocky" +- name: Install OpenJDK 17 (Ubuntu/Debian) + apt: + name: openjdk-17-jdk + state: present + update_cache: yes + become: yes + tags: + - always + when: ansible_os_family == "Debian" + +- name: Set Java home for Ubuntu/Debian + set_fact: + java_home: "{{ java_home_ubuntu }}" + when: ansible_os_family == "Debian" + +- name: Find actual Java installation path (Ubuntu/Debian) + shell: find /usr/lib/jvm -name java -type f -path "*/bin/java" 2>/dev/null | head -1 | xargs dirname | xargs dirname + register: actual_java_home + changed_when: false + when: ansible_os_family == "Debian" + +- name: Update Java home with actual path (Ubuntu/Debian) + set_fact: + java_home: "{{ actual_java_home.stdout }}" + when: ansible_os_family == "Debian" and actual_java_home.stdout != "" # NOTE: If you see a file not found error, try running rm /var/lib/alternatives/{{ item.exe }} in the target machine -- name: set {{ java_home }} as default +# On Ubuntu/Debian, the package manager already handles alternatives, so we only need to verify the path exists +- name: Verify Java path exists before setting alternatives + stat: + path: "{{ java_home }}/bin/java" + register: java_path_check + changed_when: false + failed_when: false + +- name: set {{ java_home }} as default (Ubuntu/Debian - only if path exists) alternatives: name="{{ item.exe }}" link="/usr/bin/{{ item.exe }}" @@ -46,5 +82,21 @@ - { path: "{{ java_home }}/bin", exe: 'javac' } - { path: "{{ java_home }}/bin", exe: 'javadoc' } become: yes + when: ansible_os_family == "Debian" and (java_path_check.stat.exists | default(false)) tags: - - always + - always + +- name: set {{ java_home }} as default (CentOS/Rocky) + alternatives: + name="{{ item.exe }}" + link="/usr/bin/{{ item.exe }}" + path="{{ item.path }}/{{ item.exe }}" + with_items: + - { path: "{{ java_home }}/bin", exe: 'java' } + - { path: "{{ java_home }}/bin", exe: 'keytool' } + - { path: "{{ java_home }}/bin", exe: 'javac' } + - { path: "{{ java_home }}/bin", exe: 'javadoc' } + become: yes + when: ansible_os_family == "RedHat" + tags: + - always diff --git a/dev-tools/ansible/roles/java/vars/main.yml b/dev-tools/ansible/roles/java/vars/main.yml index ef435d1107..52be23b8aa 100644 --- a/dev-tools/ansible/roles/java/vars/main.yml +++ b/dev-tools/ansible/roles/java/vars/main.yml @@ -21,9 +21,9 @@ --- #Variables associated with this role -java_home: "/usr/lib/jvm/java-11" -java_home_keycloak: "/usr/lib/jvm/java-8" -openjdk_version: "java-11-openjdk-devel-11.0.11.0.9" -openjdk_version_rocky: "java-11-openjdk-devel" -openjdk_version_rocky_keycloak: "java-1.8.0-openjdk-devel" +# Default Java home (used for CentOS/Rocky) +java_home: "/usr/lib/jvm/java-17" +java_home_ubuntu: "/usr/lib/jvm/java-17-openjdk-amd64" +openjdk_version: "java-17-openjdk-devel" +openjdk_version_rocky: "java-17-openjdk-devel" ... diff --git a/dev-tools/ansible/roles/kafka/defaults/main.yml b/dev-tools/ansible/roles/kafka/defaults/main.yml index 76a2630d2d..110c7c66dd 100644 --- a/dev-tools/ansible/roles/kafka/defaults/main.yml +++ b/dev-tools/ansible/roles/kafka/defaults/main.yml @@ -21,9 +21,10 @@ --- #Variables associated with this role -kafka_package_name: "confluent-5.3.1" -kafka_tgz_url: "https://packages.confluent.io/archive/7.9/confluent-7.9.0.zip" +kafka_package_name: "confluent-7.9.2" +kafka_tgz_url: "https://packages.confluent.io/archive/7.9/confluent-7.9.2.zip" kafka_dir: "{{ deployment_dir }}/{{ kafka_package_name }}" +kafka_symlink_name: "confluent" # Kafka related variables broker_id: "0" @@ -43,5 +44,6 @@ log_retention_check_interval: "300000" grp_initial_rebalance_delay: "0" kafka_listener_port: 9092 kafka_rest_proxy_listener_port: 8082 +zookeeper_connection_url: "{{ zookeeper_connection | default('localhost:2181') }}" ... \ No newline at end of file diff --git a/dev-tools/ansible/roles/kafka/tasks/main.yml b/dev-tools/ansible/roles/kafka/tasks/main.yml index 5b91cc822c..4614c5842e 100644 --- a/dev-tools/ansible/roles/kafka/tasks/main.yml +++ b/dev-tools/ansible/roles/kafka/tasks/main.yml @@ -35,18 +35,47 @@ become: yes # Download Kafka -- name: Download and unarchive Kafka from {{ kafka_tgz_url }} - unarchive: src="{{ kafka_tgz_url }}" - dest="{{ deployment_dir }}" - copy=no - owner="{{ user }}" - group="{{ group }}" +- name: Check if Kafka archive already exists + stat: + path: "/tmp/{{ kafka_package_name }}.zip" + register: kafka_archive + become: yes + +- name: Download Kafka archive + shell: | + cd /tmp && \ + (wget -q --no-check-certificate "{{ kafka_tgz_url }}" -O "{{ kafka_package_name }}.zip" || \ + curl -k -L "{{ kafka_tgz_url }}" -o "{{ kafka_package_name }}.zip") + args: + creates: "/tmp/{{ kafka_package_name }}.zip" + become: yes + become_user: root + when: not kafka_package.stat.exists and not kafka_archive.stat.exists + +- name: Unarchive Kafka + unarchive: + src: "/tmp/{{ kafka_package_name }}.zip" + dest: "{{ deployment_dir }}" + remote_src: yes + owner: "{{ user }}" + group: "{{ group }}" when: not kafka_package.stat.exists become: yes -# Create kafka logs directory -- name: Create kafka logs directory - file: path="{{ kafka_dir }}/logs" state=directory owner={{ user }} group={{ group }} +# Create symlink confluent -> confluent- +- name: Create Kafka symlink + file: + src: "{{ kafka_package_name }}" + dest: "{{ deployment_dir }}/{{ kafka_symlink_name }}" + state: link + owner: "{{ user }}" + group: "{{ group }}" + become: yes + when: kafka_symlink_name is defined + +# Create kafka-logs directory (matching dev server structure) +- name: Create kafka-logs directory + file: path="{{ deployment_dir }}/{{ kafka_symlink_name }}/kafka-logs" state=directory owner={{ user }} group={{ group }} become: yes # Config kafka server and start diff --git a/dev-tools/ansible/roles/kafka/templates/kafka-rest.properties.j2 b/dev-tools/ansible/roles/kafka/templates/kafka-rest.properties.j2 index 9f8d89a452..6e2ed0fa4a 100644 --- a/dev-tools/ansible/roles/kafka/templates/kafka-rest.properties.j2 +++ b/dev-tools/ansible/roles/kafka/templates/kafka-rest.properties.j2 @@ -16,7 +16,7 @@ #id=kafka-rest-test-server #schema.registry.url=http://localhost:8081 #zookeeper.connect=localhost:2181 -bootstrap.servers=PLAINTEXT://{{ groups['kafka'][0] }}:{{ kafka_listener_port }} +bootstrap.servers=PLAINTEXT://{{ ansible_default_ipv4.address }}:{{ kafka_listener_port }} listeners=http://{{ ansible_default_ipv4.address }}:{{ kafka_rest_proxy_listener_port }} # # Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center diff --git a/dev-tools/ansible/roles/kafka/templates/kafka.service.j2 b/dev-tools/ansible/roles/kafka/templates/kafka.service.j2 index afc0118546..7dc1163de9 100644 --- a/dev-tools/ansible/roles/kafka/templates/kafka.service.j2 +++ b/dev-tools/ansible/roles/kafka/templates/kafka.service.j2 @@ -6,8 +6,8 @@ Before= After=network.target [Service] -LOG_DIR={{ kafka_dir }}/logs -ExecStart={{ kafka_dir }}/bin/kafka-server-start {{ kafka_dir }}/etc/kafka/server.properties +Environment="LOG_DIR={{ deployment_dir }}/{{ kafka_symlink_name }}/logs" +ExecStart={{ deployment_dir }}/{{ kafka_symlink_name }}/bin/kafka-server-start {{ deployment_dir }}/{{ kafka_symlink_name }}/etc/kafka/server.properties Restart=on-abort [Install] diff --git a/dev-tools/ansible/roles/kafka/templates/server.properties.j2 b/dev-tools/ansible/roles/kafka/templates/server.properties.j2 index ae27bb4d38..97b8f7514f 100644 --- a/dev-tools/ansible/roles/kafka/templates/server.properties.j2 +++ b/dev-tools/ansible/roles/kafka/templates/server.properties.j2 @@ -22,13 +22,13 @@ broker.id={{ broker_id }} ############################# Socket Server Settings ############################# -# The address the socket server listens on. It will get the value returned from -# java.net.InetAddress.getCanonicalHostName() if not configured. +# The address the socket server listens on. If not configured, the host name will be equal to the value of +# java.net.InetAddress.getCanonicalHostName(), with PLAINTEXT listener name, and port 9092. # FORMAT: # listeners = listener_name://host_name:port # EXAMPLE: # listeners = PLAINTEXT://your.host.name:9092 -listeners=PLAINTEXT://{{ ansible_default_ipv4.address }}:{{ kafka_listener_port }} +#listeners=PLAINTEXT://:9092 # Hostname and port the broker will advertise to producers and consumers. If not set, # it uses the value for "listeners" if configured. Otherwise, it will use the value @@ -57,7 +57,7 @@ socket.request.max.bytes={{ socket_request_max_bytes }} ############################# Log Basics ############################# # A comma separated list of directories under which to store log files -log.dirs={{ kafka_dir }}/logs +log.dirs={{ deployment_dir }}/{{ kafka_symlink_name }}/kafka-logs # The default number of log partitions per topic. More partitions allow greater # parallelism for consumption, but this will also result in more files across @@ -123,7 +123,7 @@ log.retention.check.interval.ms={{ log_retention_check_interval }} zookeeper.connect={{ zookeeper_connection_url }} # Timeout in ms for connecting to zookeeper -zookeeper.connection.timeout.ms=6000 +zookeeper.connection.timeout.ms=18000 ##################### Confluent Metrics Reporter ####################### # Confluent Control Center and Confluent Auto Data Balancer integration @@ -189,7 +189,21 @@ confluent.license.topic.replication.factor=1 # Replication factor for the metadata topic used for authorization. Default is 3. confluent.metadata.topic.replication.factor=1 +# Replication factor for the topic used for audit logs. Default is 3. +confluent.security.event.logger.exporter.kafka.topic.replicas=1 + # Listeners for metadata server #confluent.metadata.server.listeners=http://0.0.0.0:8090 # Advertised listeners for metadata server #confluent.metadata.server.advertised.listeners=http://127.0.0.1:8090 + +############################# Confluent Data Balancer Settings ############################# + +# The Confluent Data Balancer is used to measure the load across the Kafka cluster and move data +# around as necessary. Comment out this line to disable the Data Balancer. +confluent.balancer.enable=true + +# The replication factor for the topics the Data Balancer uses to store internal state. +# For anything other than development testing, a value greater than 1 is recommended to ensure availability. +# The default value is 3. +confluent.balancer.topic.replication.factor=1 diff --git a/dev-tools/ansible/roles/keycloak/defaults/main.yml b/dev-tools/ansible/roles/keycloak/defaults/main.yml index bd766c1f54..15e51f73c0 100644 --- a/dev-tools/ansible/roles/keycloak/defaults/main.yml +++ b/dev-tools/ansible/roles/keycloak/defaults/main.yml @@ -19,27 +19,66 @@ # --- -keycloak_version: "2.5.4.Final" -keycloak_downlaod_url: "https://downloads.jboss.org/keycloak/{{keycloak_version}}/keycloak-{{keycloak_version}}.tar.gz" +keycloak_version: "24.0.0" +keycloak_downlaod_url: "https://github.com/keycloak/keycloak/releases/download/{{keycloak_version}}/keycloak-{{keycloak_version}}.tar.gz" keycloak_install_dir: "keycloak-{{keycloak_version}}" -keycloak_db_connector_name: "mysql-connector-j-9.3.0-bin.jar" +keycloak_db_connector_name: "mysql-connector-j-8.0.33.jar" +# TODO MySQL Connector/J is GPL licensed. Instead use MariaDB Connector +mysql_db_connector_download_url: "https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-j-8.0.33.tar.gz" +mysql_connector_dir_name: "mysql-connector-j-8.0.33" +mysql_connector_archive_name: "mysql-connector-j-8.0.33.tar.gz" # keycloak_ssl_keystore_file: "airavata.p12" # keycloak_ssl_keystore_file_name: "airavata.p12" # keycloak_ssl_keystore_password: "Airavata" -mysql_db_connector_download_url: "https://dev.mysql.com/get/Downloads/Connector-J/{{keycloak_db_connector_name}}.tar.gz" -keycloak_master_account_username: "username" -keycloak_master_account_password: "password" +# Set these in inventories//group_vars/all/vault.yml: +# - keycloak_master_account_username (Keycloak admin username) +# - keycloak_master_account_password (Keycloak admin password - use vault!) +# - keycloak_db_username (database username for Keycloak) +# - keycloak_db_password (database password for Keycloak - use vault!) +keycloak_master_account_username: "CHANGEME" +keycloak_master_account_password: "CHANGEME" # keycloak_server_port: "443" -keycloak_java_home: /usr/lib/jvm/java-1.8.0 +keycloak_java_home: "{{ java_home }}" +keycloak_db_vendor: "mariadb" keycloak_db_host: "localhost" keycloak_db_port: "3306" keycloak_db_schema_name: "keycloak" keycloak_db_url: "jdbc:mysql://{{keycloak_db_host}}:{{keycloak_db_port}}/{{keycloak_db_schema_name}}" -keycloak_db_username: "username" -keycloak_db_password: "password" +keycloak_quarkus_db_url: "jdbc:{{ keycloak_db_vendor }}://{{keycloak_db_host}}:{{keycloak_db_port}}/{{keycloak_db_schema_name}}" +keycloak_db_username: "CHANGEME" +keycloak_db_password: "CHANGEME" keycloak_db_pool_size: "20" +keycloak_realm_import_enabled: true +keycloak_realm_name: "default" +keycloak_realm_import_src: "realm-default.json.j2" +keycloak_realm_import_dir: "{{ user_home }}/keycloak-realms" +keycloak_realm_import_filename: "realm-default.json" +keycloak_realm_import_path: "{{ keycloak_realm_import_dir }}/{{ keycloak_realm_import_filename }}" +keycloak_realm_import_marker: "{{ keycloak_realm_import_dir }}/.imported-{{ keycloak_realm_name }}" +keycloak_logout_url: "https://{{ keycloak_vhost_servername }}/" + +# Realm client configuration - Override in inventories//group_vars/all/vault.yml +# PGA client configuration +keycloak_pga_client_secret: "CHANGEME_PGA_CLIENT_SECRET" +keycloak_pga_redirect_uris: + - "http://airavata.host:8008/callback*" + - "https://airavata.host:8009/auth/callback*" +keycloak_pga_web_origins: + - "*" + +# CS-JupyterLab client configuration +keycloak_jupyterlab_client_secret: "CHANGEME_JUPYTERLAB_CLIENT_SECRET" +keycloak_jupyterlab_redirect_uris: + - "" + - "/*" + - "http://airavata.host:20000/hub/oauth_callback" + +# CILogon identity provider configuration +keycloak_cilogon_client_id: "CHANGEME_CILOGON_CLIENT_ID" +keycloak_cilogon_client_secret: "CHANGEME_CILOGON_CLIENT_SECRET" + keycloak_vhost_servername: "changeme.org" keycloak_ssl_certificate_file: "/etc/letsencrypt/live/{{ keycloak_vhost_servername }}/cert.pem" keycloak_ssl_certificate_chain_file: "/etc/letsencrypt/live/{{ keycloak_vhost_servername }}/fullchain.pem" diff --git a/dev-tools/ansible/roles/keycloak/handlers/main.yml b/dev-tools/ansible/roles/keycloak/handlers/main.yml index 589bdabcd8..ca0ded1e44 100644 --- a/dev-tools/ansible/roles/keycloak/handlers/main.yml +++ b/dev-tools/ansible/roles/keycloak/handlers/main.yml @@ -20,7 +20,18 @@ --- -# Gracefully reload httpd +# Gracefully reload httpd (RedHat) - name: restart httpd service: name=httpd state=reloaded enabled=yes become: yes + when: ansible_os_family == "RedHat" + +# Gracefully reload apache2 (Ubuntu/Debian) +- name: restart apache2 + service: name=apache2 state=reloaded enabled=yes + become: yes + when: ansible_os_family == "Debian" + +- name: reload systemd daemon + command: systemctl daemon-reload + become: yes diff --git a/dev-tools/ansible/roles/keycloak/tasks/main.yml b/dev-tools/ansible/roles/keycloak/tasks/main.yml index 680123ff77..474b4df650 100644 --- a/dev-tools/ansible/roles/keycloak/tasks/main.yml +++ b/dev-tools/ansible/roles/keycloak/tasks/main.yml @@ -29,26 +29,31 @@ become: yes when: ansible_distribution == "Rocky" -- name: Install java - yum: name="java-1.8.0-openjdk-devel" state=present update_cache=yes +- name: Install Apache2 (Ubuntu/Debian) + apt: + name: apache2 + state: present + update_cache: yes become: yes - tags: - - always + when: ansible_os_family == "Debian" -# NOTE: If you see a file not found error, try running rm /var/lib/alternatives/{{ item.exe }} in the target machine -- name: set {{ keycloak_java_home }} as default - alternatives: - name="{{ item.exe }}" - link="/usr/bin/{{ item.exe }}" - path="{{ item.path }}/{{ item.exe }}" - with_items: - - { path: "{{ keycloak_java_home }}/bin", exe: 'java' } - - { path: "{{ keycloak_java_home }}/bin", exe: 'keytool' } - - { path: "{{ keycloak_java_home }}/bin", exe: 'javac' } - - { path: "{{ keycloak_java_home }}/bin", exe: 'javadoc' } +- name: Enable Apache2 required modules (Ubuntu/Debian) + shell: | + a2enmod proxy + a2enmod proxy_http + a2enmod ssl + a2enmod rewrite + a2enmod headers + become: yes + when: ansible_os_family == "Debian" + ignore_errors: yes + +- name: Install java (Keycloak uses system Java, already installed by java role) + debug: + msg: "Java is already installed by the java role. Keycloak will use {{ keycloak_java_home | default(java_home) }}" become: yes tags: - - always + - always - name: set selinux to permissive selinux: state=permissive policy=targeted @@ -61,6 +66,7 @@ state: yes persistent: yes become: yes + when: ansible_os_family == "RedHat" - name: Enable http/s service on public zone (for certbot verification) firewalld: service={{ item }} permanent=true state=enabled zone=public immediate=True @@ -70,13 +76,33 @@ become: yes # TODO: it seems like a virtual host config of some type is needed for the following to work -- name: copy basic virtual host file so certbot can verify domain +- name: copy basic virtual host file so certbot can verify domain (RedHat) template: src="basic-vhost.conf.j2" dest=/etc/httpd/conf.d/basic-vhost.conf backup=yes become: yes + when: ansible_os_family == "RedHat" + +- name: copy basic virtual host file so certbot can verify domain (Ubuntu/Debian) + template: src="basic-vhost.conf.j2" dest=/etc/apache2/sites-available/basic-vhost.conf backup=yes + become: yes + when: ansible_os_family == "Debian" + +- name: Enable basic virtual host (Ubuntu/Debian) + file: + src: /etc/apache2/sites-available/basic-vhost.conf + dest: /etc/apache2/sites-enabled/basic-vhost.conf + state: link + become: yes + when: ansible_os_family == "Debian" -- name: start httpd +- name: start httpd (RedHat) service: name=httpd state=started enabled=yes become: yes + when: ansible_os_family == "RedHat" + +- name: start apache2 (Ubuntu/Debian) + service: name=apache2 state=started enabled=yes + become: yes + when: ansible_os_family == "Debian" - name: check if SSL certificate exists stat: @@ -89,26 +115,89 @@ become: yes when: not stat_ssl_cert_result.stat.exists -- name: Add keycloak virtual host config that proxies to the keycloak server +- name: Add keycloak virtual host config that proxies to the keycloak server (RedHat) template: src="vhost.conf.j2" dest=/etc/httpd/conf.d/keycloak.conf backup=yes become: yes + when: ansible_os_family == "RedHat" notify: - restart httpd +- name: Add keycloak virtual host config that proxies to the keycloak server (Ubuntu/Debian) + template: src="vhost.conf.j2" dest=/etc/apache2/sites-available/keycloak.conf backup=yes + become: yes + when: ansible_os_family == "Debian" + notify: + - restart apache2 + +- name: Enable keycloak virtual host (Ubuntu/Debian) + file: + src: /etc/apache2/sites-available/keycloak.conf + dest: /etc/apache2/sites-enabled/keycloak.conf + state: link + become: yes + when: ansible_os_family == "Debian" + +- name: Restart Apache2 to apply Keycloak virtual host (Ubuntu/Debian) + service: name=apache2 state=reloaded enabled=yes + become: yes + when: ansible_os_family == "Debian" + # Download keycloak distribution -- name: Download and unarchive keycloak - unarchive: src="{{ keycloak_downlaod_url }}" - dest="{{ user_home }}" - copy=no - owner="{{ user }}" - group="{{ group }}" - creates="{{user_home}}/{{ keycloak_install_dir }}/bin/standalone.sh" +- name: Check if Keycloak is already installed (Keycloak 24+ uses Quarkus) + stat: + path: "{{user_home}}/{{ keycloak_install_dir }}/bin/kc.sh" + register: keycloak_installed_quarkus become: true become_user: "{{ user }}" + when: keycloak_version is not defined or keycloak_version.split('.')[0] | int >= 24 tags: - - always + - always + +- name: Check if Keycloak is already installed (Legacy WildFly versions) + stat: + path: "{{user_home}}/{{ keycloak_install_dir }}/bin/standalone.sh" + register: keycloak_installed_wildfly + become: true + become_user: "{{ user }}" + when: keycloak_version is defined and keycloak_version.split('.')[0] | int < 24 + tags: + - always + +- name: Download Keycloak distribution + shell: | + cd /tmp && \ + (wget -q --no-check-certificate "{{ keycloak_downlaod_url }}" -O "{{ keycloak_install_dir }}.tar.gz" || \ + curl -k -L "{{ keycloak_downlaod_url }}" -o "{{ keycloak_install_dir }}.tar.gz") && \ + tar -xzf "{{ keycloak_install_dir }}.tar.gz" -C "{{ user_home }}" && \ + chown -R {{ user }}:{{ group }} "{{ user_home }}/{{ keycloak_install_dir }}" && \ + rm -f "/tmp/{{ keycloak_install_dir }}.tar.gz" + args: + creates: "{{user_home}}/{{ keycloak_install_dir }}/bin/kc.sh" + become: true + become_user: root + when: (keycloak_version is not defined or keycloak_version.split('.')[0] | int >= 24) and not (keycloak_installed_quarkus.stat.exists | default(false)) + tags: + - always + +- name: Download Keycloak distribution (Legacy WildFly versions) + shell: | + cd /tmp && \ + (wget -q --no-check-certificate "{{ keycloak_downlaod_url }}" -O "{{ keycloak_install_dir }}.tar.gz" || \ + curl -k -L "{{ keycloak_downlaod_url }}" -o "{{ keycloak_install_dir }}.tar.gz") && \ + tar -xzf "{{ keycloak_install_dir }}.tar.gz" -C "{{ user_home }}" && \ + chown -R {{ user }}:{{ group }} "{{ user_home }}/{{ keycloak_install_dir }}" && \ + rm -f "/tmp/{{ keycloak_install_dir }}.tar.gz" + args: + creates: "{{user_home}}/{{ keycloak_install_dir }}/bin/standalone.sh" + become: true + become_user: root + when: keycloak_version is defined and keycloak_version.split('.')[0] | int < 24 and not (keycloak_installed_wildfly.stat.exists | default(false)) + tags: + - always # <---------------------------- Setup Mysql database for keycloak -------------------> +# Note: Keycloak 24+ (Quarkus) uses environment variables +# Legacy versions (< 24) use WildFly modules for JDBC driver # create folder structure - file: @@ -117,26 +206,41 @@ mode: 0755 become: true become_user: "{{ user }}" + when: keycloak_version is defined and keycloak_version.split('.')[0] | int < 24 tags: - always -- name: Download and unarchive mysql jdbc driver - unarchive: src="{{ mysql_db_connector_download_url }}" - dest="{{ user_home }}" - copy=no - owner="{{ user }}" - group="{{ group }}" - creates="{{user_home}}/{{keycloak_db_connector_name}}/{{keycloak_db_connector_name}}-bin.jar" - validate_certs=False +- name: Check if MySQL connector JAR already exists + stat: + path: "{{user_home}}/{{ mysql_connector_dir_name }}/{{ keycloak_db_connector_name }}" + register: mysql_connector_jar become: true become_user: "{{ user }}" + when: keycloak_version is defined and keycloak_version.split('.')[0] | int < 24 tags: - - always + - always + +- name: Download and unarchive mysql jdbc driver + shell: | + cd /tmp && \ + (wget -q --no-check-certificate "{{ mysql_db_connector_download_url }}" -O "{{ mysql_connector_archive_name }}" || \ + curl -k -L "{{ mysql_db_connector_download_url }}" -o "{{ mysql_connector_archive_name }}") && \ + tar -xzf "{{ mysql_connector_archive_name }}" -C "{{ user_home }}" && \ + chown -R {{ user }}:{{ group }} "{{ user_home }}/{{ mysql_connector_dir_name }}" && \ + rm -f "/tmp/{{ mysql_connector_archive_name }}" + args: + creates: "{{user_home}}/{{ mysql_connector_dir_name }}/{{ keycloak_db_connector_name }}" + become: true + become_user: root + when: keycloak_version is defined and keycloak_version.split('.')[0] | int < 24 and not (mysql_connector_jar.stat.exists | default(false)) + tags: + - always - name: move jdbc connector to keycloak module - command: mv {{user_home}}/{{keycloak_db_connector_name}}/{{keycloak_db_connector_name}}-bin.jar {{user_home}}/{{ keycloak_install_dir }}/modules/system/layers/keycloak/org/mysql/main/ + command: mv {{user_home}}/{{ mysql_connector_dir_name }}/{{ keycloak_db_connector_name }} {{user_home}}/{{ keycloak_install_dir }}/modules/system/layers/keycloak/org/mysql/main/ become: true become_user: "{{ user }}" + when: keycloak_version is defined and keycloak_version.split('.')[0] | int < 24 tags: - always @@ -149,15 +253,57 @@ mode="u=rw,g=r,o=r" become: true become_user: "{{ user }}" + when: keycloak_version is defined and keycloak_version.split('.')[0] | int < 24 tags: - always +# Keycloak 24+ uses Quarkus and requires database drivers in the providers directory +- name: Create providers directory for Keycloak 24+ (Quarkus) + file: + path: "{{user_home}}/{{ keycloak_install_dir }}/providers" + state: directory + mode: 0755 + become: true + become_user: "{{ user }}" + when: keycloak_version is not defined or keycloak_version.split('.')[0] | int >= 24 + tags: + - always + +- name: Check if MariaDB connector JAR already exists in providers (Keycloak 24+) + stat: + path: "{{user_home}}/{{ keycloak_install_dir }}/providers/mariadb-java-client.jar" + register: mariadb_connector_providers + become: true + become_user: "{{ user }}" + when: keycloak_version is not defined or keycloak_version.split('.')[0] | int >= 24 + tags: + - always + +- name: Download MariaDB Java Client for Keycloak 24+ (Quarkus) + shell: | + curl -fsSL --max-time 300 --retry 3 --retry-delay 5 \ + "https://repo1.maven.org/maven2/org/mariadb/jdbc/mariadb-java-client/3.3.3/mariadb-java-client-3.3.3.jar" \ + -o "{{user_home}}/{{ keycloak_install_dir }}/providers/mariadb-java-client.jar" && \ + chmod 644 "{{user_home}}/{{ keycloak_install_dir }}/providers/mariadb-java-client.jar" && \ + chown {{ user }}:{{ group }} "{{user_home}}/{{ keycloak_install_dir }}/providers/mariadb-java-client.jar" + args: + creates: "{{user_home}}/{{ keycloak_install_dir }}/providers/mariadb-java-client.jar" + become: true + when: > + (keycloak_version is not defined or keycloak_version.split('.')[0] | int >= 24) and + not (mariadb_connector_providers.stat.exists | default(false)) + tags: + - always + # # <---------------------------- Server Configuration --------------------------------> # Only Executed for standalone mode (SSL Configuration & MySql) -- name: copy keycloak configuration file (Standalone) +# Note: Keycloak 24.0.0+ uses Quarkus and doesn't use standalone.xml anymore +# Configuration is done via environment variables and command-line arguments +# This task is skipped for Keycloak 24.0.0+ +- name: copy keycloak configuration file (Standalone) - Legacy versions only template: > src=standalone.xml.j2 dest="{{ user_home }}/{{ keycloak_install_dir }}/standalone/configuration/standalone.xml" @@ -166,6 +312,7 @@ mode="u=rw,g=r,o=r" become: true become_user: "{{ user }}" + when: keycloak_version is not defined or keycloak_version.split('.')[0] | int < 24 tags: - standalone @@ -173,6 +320,52 @@ # <---------- setup init script for keycloak, starts the server after reboot -----------> +- name: Detect Java home if not already set (Ubuntu/Debian) + shell: find /usr/lib/jvm -name java -type f -path "*/bin/java" 2>/dev/null | head -1 | xargs dirname | xargs dirname + register: detected_java_home + changed_when: false + when: ansible_os_family == "Debian" + +- name: Verify detected Java path exists + stat: + path: "{{ detected_java_home.stdout }}/bin/java" + register: detected_java_path_check + changed_when: false + when: ansible_os_family == "Debian" and detected_java_home.stdout != "" + +- name: Set detected Java home for Keycloak (Ubuntu/Debian) + set_fact: + keycloak_java_home: "{{ detected_java_home.stdout }}" + when: ansible_os_family == "Debian" and detected_java_home.stdout != "" and (detected_java_path_check.stat.exists | default(false)) + +- name: Ensure Keycloak user can read Let's Encrypt certificates + shell: | + setfacl -m u:{{ user }}:r /etc/letsencrypt/live/{{ keycloak_vhost_servername }}/cert.pem + setfacl -m u:{{ user }}:r /etc/letsencrypt/live/{{ keycloak_vhost_servername }}/privkey.pem + setfacl -m u:{{ user }}:r /etc/letsencrypt/live/{{ keycloak_vhost_servername }}/fullchain.pem + become: yes + become_user: root + when: ansible_os_family == "Debian" + ignore_errors: yes + +- name: Copy Keycloak certificates to user-accessible location (alternative if ACL fails) + shell: | + mkdir -p {{ user_home }}/keycloak-certs + cp /etc/letsencrypt/live/{{ keycloak_vhost_servername }}/cert.pem {{ user_home }}/keycloak-certs/ + cp /etc/letsencrypt/live/{{ keycloak_vhost_servername }}/privkey.pem {{ user_home }}/keycloak-certs/ + chown -R {{ user }}:{{ group }} {{ user_home }}/keycloak-certs + chmod 600 {{ user_home }}/keycloak-certs/* + become: yes + become_user: root + when: ansible_os_family == "Debian" + register: cert_copy_result + +- name: Update Keycloak certificate paths to use user-accessible location + set_fact: + keycloak_ssl_certificate_file: "{{ user_home }}/keycloak-certs/cert.pem" + keycloak_ssl_certificate_key_file: "{{ user_home }}/keycloak-certs/privkey.pem" + when: ansible_os_family == "Debian" and cert_copy_result is succeeded + - name: copy keycloak.service systemd unit file template: src: "keycloak.service.j2" @@ -181,20 +374,77 @@ become: yes tags: - always + notify: + - reload systemd daemon + +- name: ensure Keycloak realm import directory exists + file: + path: "{{ keycloak_realm_import_dir }}" + state: directory + owner: "{{ user }}" + group: "{{ group }}" + mode: "0755" + become: yes + when: keycloak_realm_import_enabled | bool + +- name: render Keycloak realm definition for import + template: + src: "{{ keycloak_realm_import_src }}" + dest: "{{ keycloak_realm_import_path }}" + owner: "{{ user }}" + group: "{{ group }}" + mode: "0644" + become: yes + when: keycloak_realm_import_enabled | bool + +- name: import Keycloak realm definition + shell: > + {{ user_home }}/{{ keycloak_install_dir }}/bin/kc.sh import + --file {{ keycloak_realm_import_path }} + --override=true + && touch {{ keycloak_realm_import_marker }} + args: + executable: /bin/bash + chdir: "{{ user_home }}/{{ keycloak_install_dir }}" + creates: "{{ keycloak_realm_import_marker }}" + become: yes + become_user: "{{ user }}" + environment: + KC_DB: "{{ keycloak_db_vendor }}" + KC_DB_URL: "{{ keycloak_quarkus_db_url }}" + KC_DB_URL_HOST: "{{ keycloak_db_host }}" + KC_DB_URL_PORT: "{{ keycloak_db_port }}" + KC_DB_URL_DATABASE: "{{ keycloak_db_schema_name }}" + KC_DB_USERNAME: "{{ keycloak_db_username }}" + KC_DB_PASSWORD: "{{ keycloak_db_password }}" + when: + - keycloak_realm_import_enabled | bool + - keycloak_version is not defined or keycloak_version.split('.')[0] | int >= 24 # # <-------------------------Initialize a new admin for keycloak--------------------------> +# Note: Keycloak 24.0.0+ uses kc.sh and environment variables for admin setup +# The admin user is created on first start using KEYCLOAK_ADMIN and KEYCLOAK_ADMIN_PASSWORD +# For legacy versions (< 24.0.0), use add-user-keycloak.sh -- name: Add master realm admin account +- name: Add master realm admin account (Legacy versions) command: "{{user_home}}/{{ keycloak_install_dir }}/bin/add-user-keycloak.sh -r master -u {{ keycloak_master_account_username }} -p {{ keycloak_master_account_password }}" args: creates: "{{user_home}}/{{ keycloak_install_dir }}/standalone/configuration/keycloak-add-user.json" become: yes become_user: root + when: keycloak_version is not defined or (keycloak_version | default('0.0.0')).split('.')[0] | int < 24 tags: - - always + - always + +- name: Note admin user creation for Keycloak 24 + debug: + msg: "For Keycloak 24.0.0 and above, admin user will be created on first start via environment variables in systemd service" + when: keycloak_version is defined and (keycloak_version.split('.')[0] | int >= 24) + tags: + - always # <--------------------------start keycloak Identity server------------------------------> diff --git a/dev-tools/ansible/roles/keycloak/templates/basic-vhost.conf.j2 b/dev-tools/ansible/roles/keycloak/templates/basic-vhost.conf.j2 index da78c4ce26..405e102bba 100644 --- a/dev-tools/ansible/roles/keycloak/templates/basic-vhost.conf.j2 +++ b/dev-tools/ansible/roles/keycloak/templates/basic-vhost.conf.j2 @@ -1,5 +1,5 @@ - ServerName {{ groups['keycloak'][0] }} + ServerName {{ keycloak_vhost_servername }} DocumentRoot "/var/www/html" diff --git a/dev-tools/ansible/roles/keycloak/templates/keycloak.service.j2 b/dev-tools/ansible/roles/keycloak/templates/keycloak.service.j2 index da3be9f2ce..94b118fc05 100644 --- a/dev-tools/ansible/roles/keycloak/templates/keycloak.service.j2 +++ b/dev-tools/ansible/roles/keycloak/templates/keycloak.service.j2 @@ -24,9 +24,29 @@ Description=Keycloak [Service] -ExecStart={{ user_home }}/{{ keycloak_install_dir }}/bin/standalone.sh -b 0.0.0.0 +Environment="JAVA_HOME={{ keycloak_java_home }}" +{% if keycloak_version is not defined or keycloak_version.split('.')[0] | int >= 24 %} +# Keycloak 24.0.0+ uses environment variables for admin user creation +Environment="KEYCLOAK_ADMIN={{ keycloak_master_account_username }}" +Environment="KEYCLOAK_ADMIN_PASSWORD={{ keycloak_master_account_password }}" +# Database configuration for Keycloak 24+ (Quarkus) +Environment="KC_DB={{ keycloak_db_vendor }}" +Environment="KC_DB_URL={{ keycloak_quarkus_db_url }}" +Environment="KC_DB_URL_HOST={{ keycloak_db_host }}" +Environment="KC_DB_URL_PORT={{ keycloak_db_port }}" +Environment="KC_DB_URL_DATABASE={{ keycloak_db_schema_name }}" +Environment="KC_DB_USERNAME={{ keycloak_db_username }}" +Environment="KC_DB_PASSWORD={{ keycloak_db_password }}" +Environment="KC_DB_POOL_INITIAL_SIZE={{ keycloak_db_pool_size | default('20') }}" +Environment="KC_DB_POOL_MIN_SIZE={{ keycloak_db_pool_size | default('20') }}" +Environment="KC_DB_POOL_MAX_SIZE={{ keycloak_db_pool_size | default('20') }}" +{% endif %} +ExecStart={{ user_home }}/{{ keycloak_install_dir }}/bin/kc.sh start --hostname={{ keycloak_vhost_servername }} --hostname-strict=false --proxy=edge --http-enabled=false --https-certificate-file={{ keycloak_ssl_certificate_file }} --https-certificate-key-file={{ keycloak_ssl_certificate_key_file }} --https-port=8443 +WorkingDirectory={{ user_home }}/{{ keycloak_install_dir }} User={{user}} Group={{group}} +Restart=on-failure +RestartSec=10 [Install] WantedBy=multi-user.target diff --git a/dev-tools/ansible/roles/keycloak/templates/realm-default.json.j2 b/dev-tools/ansible/roles/keycloak/templates/realm-default.json.j2 new file mode 100644 index 0000000000..072c2c34ef --- /dev/null +++ b/dev-tools/ansible/roles/keycloak/templates/realm-default.json.j2 @@ -0,0 +1,2869 @@ +{ + "id": "afc8036c-62c3-462e-ae10-e1727c4bd8f7", + "realm": "default", + "displayName": "", + "displayNameHtml": "", + "notBefore": 0, + "defaultSignatureAlgorithm": "RS256", + "revokeRefreshToken": false, + "refreshTokenMaxReuse": 0, + "accessTokenLifespan": 7200, + "accessTokenLifespanForImplicitFlow": 3600, + "ssoSessionIdleTimeout": 604800, + "ssoSessionMaxLifespan": 604800, + "ssoSessionIdleTimeoutRememberMe": 0, + "ssoSessionMaxLifespanRememberMe": 0, + "offlineSessionIdleTimeout": 2592000, + "offlineSessionMaxLifespanEnabled": false, + "offlineSessionMaxLifespan": 5184000, + "clientSessionIdleTimeout": 0, + "clientSessionMaxLifespan": 0, + "clientOfflineSessionIdleTimeout": 0, + "clientOfflineSessionMaxLifespan": 0, + "accessCodeLifespan": 60, + "accessCodeLifespanUserAction": 300, + "accessCodeLifespanLogin": 1800, + "actionTokenGeneratedByAdminLifespan": 43200, + "actionTokenGeneratedByUserLifespan": 300, + "oauth2DeviceCodeLifespan": 600, + "oauth2DevicePollingInterval": 5, + "enabled": true, + "sslRequired": "external", + "registrationAllowed": false, + "registrationEmailAsUsername": false, + "rememberMe": false, + "verifyEmail": false, + "loginWithEmailAllowed": true, + "duplicateEmailsAllowed": false, + "resetPasswordAllowed": false, + "editUsernameAllowed": false, + "bruteForceProtected": false, + "permanentLockout": false, + "maxTemporaryLockouts": 0, + "maxFailureWaitSeconds": 900, + "minimumQuickLoginWaitSeconds": 60, + "waitIncrementSeconds": 60, + "quickLoginCheckMilliSeconds": 1000, + "maxDeltaTimeSeconds": 43200, + "failureFactor": 30, + "roles": { + "realm": [ + { + "id": "4cd8497d-db71-41dd-9186-f7df0c22d446", + "name": "gateway-provider", + "description": "", + "composite": false, + "clientRole": false, + "containerId": "afc8036c-62c3-462e-ae10-e1727c4bd8f7", + "attributes": {} + }, + { + "id": "b585e111-f934-43b7-b9c2-cbad0c7dc08a", + "name": "default-roles-10000000", + "description": "${role_default-roles}", + "composite": true, + "composites": { + "realm": [ + "offline_access", + "uma_authorization" + ], + "client": { + "account": [ + "view-profile", + "manage-account" + ] + } + }, + "clientRole": false, + "containerId": "afc8036c-62c3-462e-ae10-e1727c4bd8f7", + "attributes": {} + }, + { + "id": "420f07dc-c07c-4ea8-bf56-f6adf3f2bbc7", + "name": "uma_authorization", + "description": "${role_uma_authorization}", + "composite": false, + "clientRole": false, + "containerId": "afc8036c-62c3-462e-ae10-e1727c4bd8f7", + "attributes": {} + }, + { + "id": "61fafc5e-96fc-4644-98a9-94f9baf654e6", + "name": "admin", + "description": "", + "composite": false, + "clientRole": false, + "containerId": "afc8036c-62c3-462e-ae10-e1727c4bd8f7", + "attributes": {} + }, + { + "id": "1f03206b-d918-491b-a33f-ee96147b310d", + "name": "admin-read-only", + "description": "", + "composite": false, + "clientRole": false, + "containerId": "afc8036c-62c3-462e-ae10-e1727c4bd8f7", + "attributes": {} + }, + { + "id": "3f7e69dc-75d4-4388-8a34-e82d32071dc9", + "name": "offline_access", + "description": "${role_offline-access}", + "composite": false, + "clientRole": false, + "containerId": "afc8036c-62c3-462e-ae10-e1727c4bd8f7", + "attributes": {} + }, + { + "id": "ebb21957-06c9-4350-9157-576b10cc8761", + "name": "user-pending", + "description": "", + "composite": false, + "clientRole": false, + "containerId": "afc8036c-62c3-462e-ae10-e1727c4bd8f7", + "attributes": {} + }, + { + "id": "a2acdfe6-eb2a-4104-bb6a-be961e380d97", + "name": "gateway-user", + "description": "", + "composite": false, + "clientRole": false, + "containerId": "afc8036c-62c3-462e-ae10-e1727c4bd8f7", + "attributes": {} + } + ], + "client": { + "realm-management": [ + { + "id": "d14d392e-59cf-49fd-8ba9-507ffaa329cc", + "name": "manage-identity-providers", + "description": "${role_manage-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "66a7f387-fd45-4b01-9457-12692a2d5180", + "name": "realm-admin", + "description": "${role_realm-admin}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "manage-identity-providers", + "create-client", + "manage-authorization", + "impersonation", + "view-identity-providers", + "manage-clients", + "manage-users", + "query-realms", + "query-clients", + "query-users", + "view-clients", + "manage-realm", + "view-users", + "view-authorization", + "view-realm", + "query-groups", + "manage-events", + "view-events" + ] + } + }, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "d5d0c66d-b530-4fa4-af75-bf3b556873b0", + "name": "create-client", + "description": "${role_create-client}", + "composite": false, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "370cf6dc-2013-4a9b-a726-227edf3b6e04", + "name": "impersonation", + "description": "${role_impersonation}", + "composite": false, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "2fea86aa-2a95-429f-83c2-c2a9594ce050", + "name": "manage-authorization", + "description": "${role_manage-authorization}", + "composite": false, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "febffa99-60f8-4933-97c5-fe73d082802c", + "name": "view-identity-providers", + "description": "${role_view-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "658c0ab4-ded5-410a-ad62-f3c4a6a99ff4", + "name": "manage-clients", + "description": "${role_manage-clients}", + "composite": false, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "1cf0e329-8b88-4e64-95bb-e1a9e3ed2d28", + "name": "manage-users", + "description": "${role_manage-users}", + "composite": false, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "9ea990ff-0ece-463e-9792-c274aa005b3a", + "name": "query-realms", + "description": "${role_query-realms}", + "composite": false, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "6c20b3e8-97c4-4028-85c3-33531c8b8ed3", + "name": "query-clients", + "description": "${role_query-clients}", + "composite": false, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "c6aa40d9-b892-41e3-b59f-8dc332db8724", + "name": "query-users", + "description": "${role_query-users}", + "composite": false, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "24a3d19f-0d0a-4e33-b913-3166b675f5f6", + "name": "view-clients", + "description": "${role_view-clients}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-clients" + ] + } + }, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "65a3c9f0-9d28-43d8-bb95-d34eaf643493", + "name": "manage-realm", + "description": "${role_manage-realm}", + "composite": false, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "2fc992a6-c78e-4b64-9b44-b7241c993e05", + "name": "view-users", + "description": "${role_view-users}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-users", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "e0fd87a5-f6fd-4415-b38f-b767ee595812", + "name": "view-authorization", + "description": "${role_view-authorization}", + "composite": false, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "3ce7f726-99cc-4738-91d0-0d2d5559e013", + "name": "view-realm", + "description": "${role_view-realm}", + "composite": false, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "43cb7af1-22b5-49f2-a98a-2ac0b8ad887f", + "name": "query-groups", + "description": "${role_query-groups}", + "composite": false, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "03528436-9a19-41c6-bb7b-29a504fe7fa8", + "name": "manage-events", + "description": "${role_manage-events}", + "composite": false, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + }, + { + "id": "8586e2c7-b3a5-4f64-beb6-5f61617f661e", + "name": "view-events", + "description": "${role_view-events}", + "composite": false, + "clientRole": true, + "containerId": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "attributes": {} + } + ], + "cybershuttle-agent": [], + "cs-jupyterlab": [ + { + "id": "bc3da200-4725-43cc-abdc-efee7c26a748", + "name": "uma_protection", + "composite": false, + "clientRole": true, + "containerId": "ac2469cf-1760-4d47-9079-7306fee96ae4", + "attributes": {} + } + ], + "security-admin-console": [], + "admin-cli": [], + "account-console": [], + "pga": [ + { + "id": "d8d76309-d081-4159-b2cd-d9ca93eb7d02", + "name": "uma_protection", + "composite": false, + "clientRole": true, + "containerId": "5e2398e0-3498-4da3-9262-4f2dcc7448fa", + "attributes": {} + }, + { + "id": "f8051cd8-10cb-44e6-8826-d323daa236d1", + "name": "gateway-provider", + "description": "", + "composite": false, + "clientRole": true, + "containerId": "5e2398e0-3498-4da3-9262-4f2dcc7448fa", + "attributes": {} + }, + { + "id": "fb2c5f47-09e2-4f4b-b858-625f3c5442cd", + "name": "user-pending", + "description": "", + "composite": false, + "clientRole": true, + "containerId": "5e2398e0-3498-4da3-9262-4f2dcc7448fa", + "attributes": {} + }, + { + "id": "c7d75283-b7c3-4b93-8804-9ce55bccf74c", + "name": "admin", + "description": "", + "composite": false, + "clientRole": true, + "containerId": "5e2398e0-3498-4da3-9262-4f2dcc7448fa", + "attributes": {} + }, + { + "id": "da796582-cbf2-4b23-a31d-5fc9b4010bb0", + "name": "admin-read-only", + "description": "", + "composite": false, + "clientRole": true, + "containerId": "5e2398e0-3498-4da3-9262-4f2dcc7448fa", + "attributes": {} + }, + { + "id": "42660438-3a37-466f-b748-d25a25ff9082", + "name": "gateway-user", + "description": "", + "composite": false, + "clientRole": true, + "containerId": "5e2398e0-3498-4da3-9262-4f2dcc7448fa", + "attributes": {} + } + ], + "broker": [ + { + "id": "dd71ed3a-bb48-41b6-9dae-e81170c4c445", + "name": "read-token", + "description": "${role_read-token}", + "composite": false, + "clientRole": true, + "containerId": "af99dd7f-6d3a-4fec-93c0-8deab50edf0e", + "attributes": {} + } + ], + "account": [ + { + "id": "3fa60a39-7c55-434e-b602-3789dd70ec15", + "name": "view-applications", + "description": "${role_view-applications}", + "composite": false, + "clientRole": true, + "containerId": "c1602103-6c60-4e27-a7ed-c2c21d7801f2", + "attributes": {} + }, + { + "id": "2f3dc94f-9347-49cf-914e-dc3615e550e1", + "name": "view-profile", + "description": "${role_view-profile}", + "composite": false, + "clientRole": true, + "containerId": "c1602103-6c60-4e27-a7ed-c2c21d7801f2", + "attributes": {} + }, + { + "id": "501821b9-9fd0-42df-b89d-28375acfcbaa", + "name": "manage-account-links", + "description": "${role_manage-account-links}", + "composite": false, + "clientRole": true, + "containerId": "c1602103-6c60-4e27-a7ed-c2c21d7801f2", + "attributes": {} + }, + { + "id": "c9752755-7d07-40b7-bf61-da14c3d524f2", + "name": "view-groups", + "description": "${role_view-groups}", + "composite": false, + "clientRole": true, + "containerId": "c1602103-6c60-4e27-a7ed-c2c21d7801f2", + "attributes": {} + }, + { + "id": "de86f028-34c5-422e-a8a0-0508bd5071ed", + "name": "view-consent", + "description": "${role_view-consent}", + "composite": false, + "clientRole": true, + "containerId": "c1602103-6c60-4e27-a7ed-c2c21d7801f2", + "attributes": {} + }, + { + "id": "c5b77ca2-7619-4ad9-994d-0bd6ba8e8747", + "name": "manage-account", + "description": "${role_manage-account}", + "composite": true, + "composites": { + "client": { + "account": [ + "manage-account-links" + ] + } + }, + "clientRole": true, + "containerId": "c1602103-6c60-4e27-a7ed-c2c21d7801f2", + "attributes": {} + }, + { + "id": "fd21c288-87c3-414c-9152-ea3f5a23b2ca", + "name": "manage-consent", + "description": "${role_manage-consent}", + "composite": true, + "composites": { + "client": { + "account": [ + "view-consent" + ] + } + }, + "clientRole": true, + "containerId": "c1602103-6c60-4e27-a7ed-c2c21d7801f2", + "attributes": {} + }, + { + "id": "ea81e9c3-3f7b-4cda-901b-4bbe6a30e5e7", + "name": "delete-account", + "description": "${role_delete-account}", + "composite": false, + "clientRole": true, + "containerId": "c1602103-6c60-4e27-a7ed-c2c21d7801f2", + "attributes": {} + } + ] + } + }, + "groups": [], + "defaultRole": { + "id": "b585e111-f934-43b7-b9c2-cbad0c7dc08a", + "name": "default-roles-10000000", + "description": "${role_default-roles}", + "composite": true, + "clientRole": false, + "containerId": "afc8036c-62c3-462e-ae10-e1727c4bd8f7" + }, + "requiredCredentials": [ + "password" + ], + "otpPolicyType": "totp", + "otpPolicyAlgorithm": "HmacSHA1", + "otpPolicyInitialCounter": 0, + "otpPolicyDigits": 6, + "otpPolicyLookAheadWindow": 1, + "otpPolicyPeriod": 30, + "otpPolicyCodeReusable": false, + "otpSupportedApplications": [ + "totpAppFreeOTPName", + "totpAppGoogleName", + "totpAppMicrosoftAuthenticatorName" + ], + "localizationTexts": {}, + "webAuthnPolicyRpEntityName": "keycloak", + "webAuthnPolicySignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyRpId": "", + "webAuthnPolicyAttestationConveyancePreference": "not specified", + "webAuthnPolicyAuthenticatorAttachment": "not specified", + "webAuthnPolicyRequireResidentKey": "not specified", + "webAuthnPolicyUserVerificationRequirement": "not specified", + "webAuthnPolicyCreateTimeout": 0, + "webAuthnPolicyAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyAcceptableAaguids": [], + "webAuthnPolicyExtraOrigins": [], + "webAuthnPolicyPasswordlessRpEntityName": "keycloak", + "webAuthnPolicyPasswordlessSignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyPasswordlessRpId": "", + "webAuthnPolicyPasswordlessAttestationConveyancePreference": "not specified", + "webAuthnPolicyPasswordlessAuthenticatorAttachment": "not specified", + "webAuthnPolicyPasswordlessRequireResidentKey": "not specified", + "webAuthnPolicyPasswordlessUserVerificationRequirement": "not specified", + "webAuthnPolicyPasswordlessCreateTimeout": 0, + "webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyPasswordlessAcceptableAaguids": [], + "webAuthnPolicyPasswordlessExtraOrigins": [], + "users": [ + { + "id": "33865abf-b336-4f77-b370-c2aaadcefaa8", + "username": "default-admin", + "firstName": "admin", + "lastName": "admin", + "email": "default-admin@default", + "emailVerified": true, + "createdTimestamp": 1741788577569, + "enabled": true, + "totp": false, + "credentials": [ + { + "id": "21ba2cc3-1794-4f87-8df4-a7350cc570ca", + "type": "password", + "userLabel": "My password", + "createdDate": 1741788838245, + "secretData": "{\"value\":\"qQ+RHT09vJb1Mv4snElCbOh67CM7cO8r2oFX5UtZunk33EG/uplFAOTIeklRMU5HydfeL1u8gisa9ui+8e2A2g==\",\"salt\":\"tbme6ZolnVOjWqLtWTmzSA==\",\"additionalParameters\":{}}", + "credentialData": "{\"hashIterations\":210000,\"algorithm\":\"pbkdf2-sha512\",\"additionalParameters\":{}}" + } + ], + "disableableCredentialTypes": [], + "requiredActions": [], + "realmRoles": [ + "default-roles-10000000" + ], + "notBefore": 0, + "groups": [] + }, + { + "id": "e3fb714e-9ed6-4d26-a403-781e71bbb025", + "username": "service-account-cs-jupyterlab", + "emailVerified": false, + "createdTimestamp": 1733075288051, + "enabled": true, + "totp": false, + "serviceAccountClientId": "cs-jupyterlab", + "credentials": [], + "disableableCredentialTypes": [], + "requiredActions": [], + "realmRoles": [ + "default-roles-10000000" + ], + "clientRoles": { + "cs-jupyterlab": [ + "uma_protection" + ] + }, + "notBefore": 0, + "groups": [] + }, + { + "id": "e9d5a7b9-c093-4916-a331-12fbc9101c70", + "username": "service-account-pga", + "emailVerified": false, + "createdTimestamp": 1726317784923, + "enabled": true, + "totp": false, + "serviceAccountClientId": "pga", + "credentials": [], + "disableableCredentialTypes": [], + "requiredActions": [], + "realmRoles": [ + "default-roles-10000000" + ], + "clientRoles": { + "realm-management": [ + "manage-users" + ], + "pga": [ + "uma_protection" + ] + }, + "notBefore": 0, + "groups": [] + } + ], + "scopeMappings": [ + { + "clientScope": "offline_access", + "roles": [ + "offline_access" + ] + } + ], + "clientScopeMappings": { + "account": [ + { + "client": "account-console", + "roles": [ + "manage-account", + "view-groups" + ] + } + ] + }, + "clients": [ + { + "id": "c1602103-6c60-4e27-a7ed-c2c21d7801f2", + "clientId": "account", + "name": "${client_account}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/default/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/default/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "post.logout.redirect.uris": "+" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "def3d25e-919c-4b1b-b3fd-0a976a33e4b6", + "clientId": "account-console", + "name": "${client_account-console}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/default/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/default/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "post.logout.redirect.uris": "+", + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "a2d8962f-2ac2-4fa5-b42e-f37990a4d098", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + } + ], + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "4a5fd0fd-6842-4528-8fea-ca0727dce936", + "clientId": "admin-cli", + "name": "${client_admin-cli}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": false, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "post.logout.redirect.uris": "+" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "af99dd7f-6d3a-4fec-93c0-8deab50edf0e", + "clientId": "broker", + "name": "${client_broker}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "post.logout.redirect.uris": "+" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "ac2469cf-1760-4d47-9079-7306fee96ae4", + "clientId": "cs-jupyterlab", + "name": "JupyterLab", + "description": "", + "rootUrl": "", + "adminUrl": "", + "baseUrl": "http://localhost:8080/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "{{ keycloak_jupyterlab_client_secret }}", + "redirectUris": {{ keycloak_jupyterlab_redirect_uris | to_json }}, + "webOrigins": [ + "/*" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": true, + "authorizationServicesEnabled": true, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "client.secret.creation.time": "1741725835", + "post.logout.redirect.uris": "+", + "oauth2.device.authorization.grant.enabled": "false", + "backchannel.logout.revoke.offline.tokens": "false", + "use.refresh.tokens": "true", + "oidc.ciba.grant.enabled": "false", + "client.use.lightweight.access.token.enabled": "false", + "backchannel.logout.session.required": "true", + "client_credentials.use_refresh_token": "false", + "acr.loa.map": "{}", + "require.pushed.authorization.requests": "false", + "tls.client.certificate.bound.access.tokens": "false", + "display.on.consent.screen": "false", + "token.response.type.bearer.lower-case": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "protocolMappers": [ + { + "id": "8674e3d9-f43c-4f89-a290-2666ed0567c1", + "name": "Client IP Address", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientAddress", + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientAddress", + "jsonType.label": "String" + } + }, + { + "id": "b0fdec17-a04b-48c7-870b-ba71c0bb6680", + "name": "Client ID", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "client_id", + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "client_id", + "jsonType.label": "String" + } + }, + { + "id": "f8031842-1fd7-41ac-b5e8-b22330cdcdda", + "name": "Client Host", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientHost", + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientHost", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "preferred_username", + "microprofile-jwt" + ], + "authorizationSettings": { + "allowRemoteResourceManagement": true, + "policyEnforcementMode": "ENFORCING", + "resources": [], + "policies": [], + "scopes": [], + "decisionStrategy": "UNANIMOUS" + } + }, + { + "id": "f4c30fed-2f14-471d-a922-b5ad262273f2", + "clientId": "cybershuttle-agent", + "name": "CyberShuttle Agent", + "description": "", + "rootUrl": "", + "adminUrl": "", + "baseUrl": "http://airavata.host:8009/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/*" + ], + "webOrigins": [ + "/*" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": false, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": true, + "protocol": "openid-connect", + "attributes": { + "post.logout.redirect.uris": "+", + "oauth2.device.authorization.grant.enabled": "true", + "backchannel.logout.revoke.offline.tokens": "false", + "use.refresh.tokens": "true", + "oidc.ciba.grant.enabled": "false", + "client.use.lightweight.access.token.enabled": "false", + "backchannel.logout.session.required": "true", + "client_credentials.use_refresh_token": "false", + "acr.loa.map": "{}", + "require.pushed.authorization.requests": "false", + "tls.client.certificate.bound.access.tokens": "false", + "display.on.consent.screen": "false", + "token.response.type.bearer.lower-case": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "5e2398e0-3498-4da3-9262-4f2dcc7448fa", + "clientId": "pga", + "name": "Cybeshuttle Client", + "description": "Client For Cybershuttle Services", + "rootUrl": "", + "adminUrl": "", + "baseUrl": "", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "{{ keycloak_pga_client_secret }}", + "redirectUris": {{ keycloak_pga_redirect_uris | to_json }}, + "webOrigins": {{ keycloak_pga_web_origins | to_json }}, + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": true, + "authorizationServicesEnabled": true, + "publicClient": false, + "frontchannelLogout": true, + "protocol": "openid-connect", + "attributes": { + "oidc.ciba.grant.enabled": "false", + "client.secret.creation.time": "1741724922", + "backchannel.logout.session.required": "true", + "frontchannel.logout.url": "http://airavata.host:8009/", + "post.logout.redirect.uris": "+##http://airavata.host:8009/", + "display.on.consent.screen": "false", + "oauth2.device.authorization.grant.enabled": "true", + "backchannel.logout.revoke.offline.tokens": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "protocolMappers": [ + { + "id": "f15a7de0-0c1e-40d8-bd05-c1aaf0deb3e1", + "name": "Client IP Address", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientAddress", + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientAddress", + "jsonType.label": "String" + } + }, + { + "id": "a0956d0b-e5c4-4d9a-aebf-89efa6881438", + "name": "Client ID", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "client_id", + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "client_id", + "jsonType.label": "String" + } + }, + { + "id": "6863299e-7d4f-43f4-8d0e-fc8cd4a8ceac", + "name": "Client Host", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientHost", + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientHost", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "preferred_username", + "microprofile-jwt" + ], + "authorizationSettings": { + "allowRemoteResourceManagement": true, + "policyEnforcementMode": "ENFORCING", + "resources": [], + "policies": [], + "scopes": [], + "decisionStrategy": "UNANIMOUS" + } + }, + { + "id": "be9976ab-8e62-4d5a-8176-b6efaba2e1bf", + "clientId": "realm-management", + "name": "${client_realm-management}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "post.logout.redirect.uris": "+" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "cedda658-7913-4763-89b8-71d1d1794c1c", + "clientId": "security-admin-console", + "name": "${client_security-admin-console}", + "rootUrl": "${authAdminUrl}", + "baseUrl": "/admin/default/console/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/admin/default/console/*" + ], + "webOrigins": [ + "+" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "post.logout.redirect.uris": "+", + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "f8a3200c-fed5-4d3e-9fc7-80c23c458056", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + } + ], + "clientScopes": [ + { + "id": "a8292b23-f927-4a5c-a432-2f0ae8867105", + "name": "roles", + "description": "OpenID Connect scope for add user roles to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "false", + "gui.order": "", + "consent.screen.text": "${rolesScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "bfd13362-6c5d-405c-b7c0-bd37063ca9f9", + "name": "client roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-client-role-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "multivalued": "true", + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "resource_access.${client_id}.roles", + "jsonType.label": "String" + } + }, + { + "id": "be26563f-80a7-43d7-bf9f-d71205ec53be", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": { + "access.token.claim": "true", + "introspection.token.claim": "true" + } + }, + { + "id": "9565b5e4-dfbb-43bc-aa8d-3e845c188669", + "name": "realm roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "multivalued": "true", + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "34779656-fdf3-41c5-a963-d1e2b533b8f0", + "name": "phone", + "description": "OpenID Connect built-in scope: phone", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${phoneScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "6d5c53bd-a91a-4066-baba-52de991b5d53", + "name": "phone number", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "phoneNumber", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number", + "jsonType.label": "String" + } + }, + { + "id": "a4f0d91a-e59e-4794-b71c-726413ba5be8", + "name": "phone number verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "phoneNumberVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "8240c399-c132-49a0-8e41-48266e89b2ce", + "name": "address", + "description": "OpenID Connect built-in scope: address", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${addressScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "cbd43ec1-4467-4148-a435-081cd1c0d161", + "name": "address", + "protocol": "openid-connect", + "protocolMapper": "oidc-address-mapper", + "consentRequired": false, + "config": { + "user.attribute.formatted": "formatted", + "user.attribute.country": "country", + "introspection.token.claim": "true", + "user.attribute.postal_code": "postal_code", + "userinfo.token.claim": "true", + "user.attribute.street": "street", + "id.token.claim": "true", + "user.attribute.region": "region", + "access.token.claim": "true", + "user.attribute.locality": "locality" + } + } + ] + }, + { + "id": "7d17e91f-6691-4f1d-9f6c-7a6a7ae0b5ae", + "name": "email", + "description": "OpenID Connect built-in scope: email", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false", + "gui.order": "", + "consent.screen.text": "${emailScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "c0d4e90d-2c57-4ed5-9e82-01a6d1d16cf0", + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "email", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email", + "jsonType.label": "String" + } + }, + { + "id": "c7e78134-c7dd-45a9-a921-cc5dbdc9fc68", + "name": "email verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "emailVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "c0509e2e-01ae-49d6-8d24-0e86628a85aa", + "name": "preferred_username", + "description": "preferred_username", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "gui.order": "", + "consent.screen.text": "" + }, + "protocolMappers": [ + { + "id": "e56aa26c-4614-43bb-a96b-56c1ad959e45", + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "email", + "id.token.claim": "true", + "lightweight.claim": "false", + "access.token.claim": "true", + "claim.name": "email", + "jsonType.label": "String" + } + }, + { + "id": "b50ae18b-52e8-44f6-ad4d-ad596de89cfd", + "name": "username", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "lightweight.claim": "false", + "access.token.claim": "true", + "claim.name": "preferred_username", + "jsonType.label": "String" + } + }, + { + "id": "0a93294b-2a7c-4098-af47-4faaa535c8d4", + "name": "ClientId", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientId", + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "id.token.claim": "true", + "lightweight.claim": "false", + "access.token.claim": "true", + "claim.name": "clientId", + "jsonType.label": "String", + "access.tokenResponse.claim": "false" + } + }, + { + "id": "37b8b958-2396-4e2c-871c-f56f6ad393aa", + "name": "family name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "lastName", + "id.token.claim": "true", + "lightweight.claim": "false", + "access.token.claim": "true", + "claim.name": "family_name", + "jsonType.label": "String" + } + }, + { + "id": "d779bc1f-5c49-46c6-ae84-1d9e419984ba", + "name": "given name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "firstName", + "id.token.claim": "true", + "lightweight.claim": "false", + "access.token.claim": "true", + "claim.name": "given_name", + "jsonType.label": "String" + } + }, + { + "id": "75c6a588-0993-446e-90e0-52046505959e", + "name": "full name", + "protocol": "openid-connect", + "protocolMapper": "oidc-full-name-mapper", + "consentRequired": false, + "config": { + "id.token.claim": "true", + "lightweight.claim": "false", + "access.token.claim": "true", + "introspection.token.claim": "true", + "userinfo.token.claim": "false" + } + } + ] + }, + { + "id": "33b54357-7cdd-494c-9806-6e6d59f6af69", + "name": "role_list", + "description": "SAML role list", + "protocol": "saml", + "attributes": { + "consent.screen.text": "${samlRoleListScopeConsentText}", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "c56cb073-719d-4f74-8a23-b2105804ee80", + "name": "role list", + "protocol": "saml", + "protocolMapper": "saml-role-list-mapper", + "consentRequired": false, + "config": { + "single": "false", + "attribute.nameformat": "Basic", + "attribute.name": "Role" + } + } + ] + }, + { + "id": "658e931d-3b55-464e-b199-48e9b43f9c3f", + "name": "profile", + "description": "OpenID Connect built-in scope: profile", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false", + "gui.order": "", + "consent.screen.text": "${profileScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "a352424d-d48b-4e7e-9394-c769f9542605", + "name": "website", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "website", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "website", + "jsonType.label": "String" + } + }, + { + "id": "3710a2dd-969f-4562-89af-55e31e30f565", + "name": "middle name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "middleName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "middle_name", + "jsonType.label": "String" + } + }, + { + "id": "862fa550-a9df-40e5-8fa3-473c45de5e59", + "name": "zoneinfo", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "zoneinfo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "zoneinfo", + "jsonType.label": "String" + } + }, + { + "id": "0ae8854b-1589-401c-b0e0-24e423d8e723", + "name": "given name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "firstName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "given_name", + "jsonType.label": "String" + } + }, + { + "id": "a72f9f11-04bb-4905-81fe-7b23b7446e20", + "name": "gender", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "gender", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "gender", + "jsonType.label": "String" + } + }, + { + "id": "eaded814-dc28-450e-95ca-fbd2712e6a40", + "name": "profile", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "profile", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "profile", + "jsonType.label": "String" + } + }, + { + "id": "652dd31b-d4d8-49e9-8f53-125404b2a1f1", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + }, + { + "id": "b20740b4-9fea-4035-b3af-cf0a40852b2f", + "name": "birthdate", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "birthdate", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "birthdate", + "jsonType.label": "String" + } + }, + { + "id": "7d36aacc-aa67-4ccb-98b0-b02a111c7dea", + "name": "username", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "preferred_username", + "jsonType.label": "String" + } + }, + { + "id": "4ed1750d-d48a-4005-8398-2c34f0c6548e", + "name": "full name", + "protocol": "openid-connect", + "protocolMapper": "oidc-full-name-mapper", + "consentRequired": false, + "config": { + "id.token.claim": "true", + "access.token.claim": "true", + "introspection.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "id": "4b111e20-11ee-4429-be4d-93414b08e506", + "name": "updated at", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "updatedAt", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "updated_at", + "jsonType.label": "long" + } + }, + { + "id": "7a7e5200-ae1c-44dc-a5a7-5546391929b5", + "name": "family name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "lastName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "family_name", + "jsonType.label": "String" + } + }, + { + "id": "d353e69e-7d1e-4c41-9fdd-f76a59dacf2b", + "name": "picture", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "picture", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "picture", + "jsonType.label": "String" + } + }, + { + "id": "a0e4a0a3-1428-46a5-a2a8-66480b1ecf9f", + "name": "nickname", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "nickname", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "nickname", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "af72f98c-7086-4e99-97ad-2cdfe9f46cd4", + "name": "offline_access", + "description": "OpenID Connect built-in scope: offline_access", + "protocol": "openid-connect", + "attributes": { + "consent.screen.text": "${offlineAccessScopeConsentText}", + "display.on.consent.screen": "true" + } + }, + { + "id": "c98af4d7-e60f-469d-b76b-c8673b3d641c", + "name": "acr", + "description": "OpenID Connect scope for add acr (authentication context class reference) to the token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "d000afcc-348f-414e-8e18-fbf9dd73bec0", + "name": "acr loa level", + "protocol": "openid-connect", + "protocolMapper": "oidc-acr-mapper", + "consentRequired": false, + "config": { + "id.token.claim": "true", + "access.token.claim": "true", + "introspection.token.claim": "true", + "userinfo.token.claim": "true" + } + } + ] + }, + { + "id": "9bc461ac-7b67-48ae-af92-03b51bdd2336", + "name": "microprofile-jwt", + "description": "Microprofile - JWT built-in scope", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "075ba449-99e1-49c5-8642-755d9fd1ca08", + "name": "upn", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "upn", + "jsonType.label": "String" + } + }, + { + "id": "4d1db13c-a944-4205-95fc-6b9c184e3cc4", + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "multivalued": "true", + "userinfo.token.claim": "true", + "user.attribute": "foo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "groups", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "162a761a-4a0b-4c30-a388-1ee5df06eb8e", + "name": "web-origins", + "description": "OpenID Connect scope for add allowed web origins to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "false", + "consent.screen.text": "" + }, + "protocolMappers": [ + { + "id": "2ecc5181-9142-45b5-b11a-4d7936f9e38e", + "name": "allowed web origins", + "protocol": "openid-connect", + "protocolMapper": "oidc-allowed-origins-mapper", + "consentRequired": false, + "config": { + "access.token.claim": "true", + "introspection.token.claim": "true" + } + } + ] + } + ], + "defaultDefaultClientScopes": [ + "web-origins", + "role_list", + "profile", + "email", + "roles", + "acr" + ], + "defaultOptionalClientScopes": [ + "phone", + "address", + "microprofile-jwt", + "offline_access", + "preferred_username" + ], + "browserSecurityHeaders": { + "contentSecurityPolicyReportOnly": "", + "xContentTypeOptions": "nosniff", + "referrerPolicy": "no-referrer", + "xRobotsTag": "none", + "xFrameOptions": "SAMEORIGIN", + "contentSecurityPolicy": "frame-src 'self'; frame-ancestors 'self'; object-src 'none';", + "xXSSProtection": "1; mode=block", + "strictTransportSecurity": "max-age=31536000; includeSubDomains" + }, + "smtpServer": {}, + "loginTheme": "custom-theme", + "accountTheme": "", + "adminTheme": "", + "emailTheme": "", + "eventsEnabled": false, + "eventsListeners": [ + "jboss-logging" + ], + "enabledEventTypes": [], + "adminEventsEnabled": false, + "adminEventsDetailsEnabled": false, + "identityProviders": [ + { + "alias": "oidc", + "displayName": "CILogon", + "internalId": "f0427047-bcb0-414d-9e9a-dc97b7cddefa", + "providerId": "oidc", + "enabled": true, + "updateProfileFirstLoginMode": "on", + "trustEmail": true, + "storeToken": true, + "addReadTokenRoleOnCreate": true, + "authenticateByDefault": false, + "linkOnly": false, + "firstBrokerLoginFlowAlias": "first broker login", + "config": { + "acceptsPromptNoneForwardFromClient": "false", + "tokenUrl": "https://cilogon.org/oauth2/token", + "isAccessTokenJWT": "false", + "filteredByClaim": "false", + "backchannelSupported": "false", + "issuer": "https://cilogon.org", + "loginHint": "false", + "clientAuthMethod": "client_secret_post", + "syncMode": "IMPORT", + "clientSecret": "{{ keycloak_cilogon_client_secret }}", + "allowedClockSkew": "0", + "defaultScope": "openid profile email org.cilogon.userinfo", + "guiOrder": "1", + "hideOnLoginPage": "false", + "userInfoUrl": "https://cilogon.org/oauth2/userinfo", + "validateSignature": "false", + "clientId": "{{ keycloak_cilogon_client_id }}", + "uiLocales": "false", + "disableNonce": "false", + "sendClientIdOnLogout": "false", + "pkceEnabled": "false", + "forwardParameters": "kc_idp_hint", + "authorizationUrl": "https://cilogon.org/authorize", + "disableUserInfo": "false", + "logoutUrl": "https://cilogon.org/logout", + "sendIdTokenOnLogout": "true", + "passMaxAge": "false" + } + } + ], + "identityProviderMappers": [ + { + "id": "59cabb48-742d-471c-9bb5-8741c98675ad", + "name": "family_name", + "identityProviderAlias": "oidc", + "identityProviderMapper": "oidc-user-attribute-idp-mapper", + "config": { + "syncMode": "INHERIT", + "claim": "family_name", + "user.attribute": "lastName" + } + }, + { + "id": "6d5651e9-75cd-4c70-9bbc-6f2acc5496ab", + "name": "given_name", + "identityProviderAlias": "oidc", + "identityProviderMapper": "oidc-user-attribute-idp-mapper", + "config": { + "syncMode": "INHERIT", + "claim": "given_name", + "user.attribute": "firstName" + } + } + ], + "components": { + "org.keycloak.services.clientregistration.policy.ClientRegistrationPolicy": [ + { + "id": "d446bfe5-46d7-4b5a-a569-41268f1f0e87", + "name": "Trusted Hosts", + "providerId": "trusted-hosts", + "subType": "anonymous", + "subComponents": {}, + "config": { + "host-sending-registration-request-must-match": [ + "true" + ], + "client-uris-must-match": [ + "true" + ] + } + }, + { + "id": "d6c586ad-07ab-4a3e-8f61-9ee3d4f7e03f", + "name": "Max Clients Limit", + "providerId": "max-clients", + "subType": "anonymous", + "subComponents": {}, + "config": { + "max-clients": [ + "200" + ] + } + }, + { + "id": "fcf2e8da-427f-4232-8e02-cf08151c0211", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "saml-user-property-mapper", + "saml-user-attribute-mapper", + "oidc-full-name-mapper", + "saml-role-list-mapper", + "oidc-address-mapper", + "oidc-sha256-pairwise-sub-mapper", + "oidc-usermodel-property-mapper", + "oidc-usermodel-attribute-mapper" + ] + } + }, + { + "id": "9d32bc1e-fa04-4ab7-8911-cb344ff8b5c8", + "name": "Consent Required", + "providerId": "consent-required", + "subType": "anonymous", + "subComponents": {}, + "config": {} + }, + { + "id": "eff493d8-9108-420f-88ca-522e9df73f46", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "saml-role-list-mapper", + "saml-user-property-mapper", + "saml-user-attribute-mapper", + "oidc-usermodel-property-mapper", + "oidc-address-mapper", + "oidc-usermodel-attribute-mapper", + "oidc-full-name-mapper", + "oidc-sha256-pairwise-sub-mapper" + ] + } + }, + { + "id": "64e7e77a-3489-45bd-b440-e1c42941b22a", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + }, + { + "id": "b3315017-dfb7-4e2f-a682-f9278ae9008a", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + }, + { + "id": "35fa6e56-7042-4df8-bdda-feeeb6b39c45", + "name": "Full Scope Disabled", + "providerId": "scope", + "subType": "anonymous", + "subComponents": {}, + "config": {} + } + ], + "org.keycloak.keys.KeyProvider": [ + { + "id": "33a8177a-0278-4ce3-abee-a4394ec04aed", + "name": "rsa-enc-generated", + "providerId": "rsa-enc-generated", + "subComponents": {}, + "config": { + "privateKey": [ + "MIIEowIBAAKCAQEArMNxvbCXCLZBiqnNEcZ+XpGJVvKXnGVOvc6P4CT4IDDqjuSUWbpGAECPiygYjlUDRkNw5qY0Z8Foo4g0/wBOBvEMotkBEW8JiT9ND1JZAFcqlVksSMdE9YvmrZSoteSs5wSKHxvh0903cLEtRUmwlq1AVt6Eux90ime5iuBbLmwskk882SUcxgRsDp4iocrBZ99bl6oq10D+7h6JyNS0k7fh9o7q5F4fsqENjXvoPASRk1R0Q3pJPT7Uih/wetm4iv9S+jqmoFccYjBnamy7U/NRF2XauTLr0Xj/S2WRPb+eDF59onPyz13Cr/jZC5YzaKqjFGBReW+c3YJ5ETDdWQIDAQABAoIBAAkRpGec/LyuFpAxHCz18r3DnYZFK0zAK6s+i+JsBmNoNfPkz13Lb6/FM9PN+cYE8+xND4DoEiGtLzFAbenB6hamsi9dPVddMQ61ljW68KWaLcfTu8WhQj8yhhwwDNApjiL9Y8PAyrC8sNOXVWVI+j5an2FAAo8xFkTKr5x47QhperQoOyRglNgxaPhZTv98EFKzTFGyyC3zJsffNyyRn2KRATGH3G2wQEIqcaWiIcqypPruwdNy/C42SRLhWTLCW4iUiZvACPBXhTktb/BaJ8dE6h+PAMM1leeor6JolDB62jE8NuBj3sLstnqjXrXlx5xKhK4OWkr3IGu5zibeCQUCgYEA0vSZYab8KyEd1MOPYVUHMxuLrS0WYb9fe+nLulL1G4DiTityUPkEKh0SwJEwUrgI1Zzfe0lRAhmpZSRHq9qrgYtB23JorjyKWtugxqd6Ri0WgA1p3DDkn9lakxDBTEESmaM/fQl65/Jbw9WrrGvDP/L4qvOM6ggXFkj+l+TiKRUCgYEA0acqa7xKmXGvfpZclYXom/03dWqDkpGA1PZyGQPsvD0EJADojWCFLcD1/afCe7m244a1NYjAQKtCl4zOTzi5MmiaA3r5J0K+qxzO1VTdRwCsmtS7eQT1kbHdstvXjS6an9WtpOTkpFZSYzXEOmOy6mBF8SpBV5s1j1pv+7BD7DUCgYEAupNhEPAqeU7J3oKzzibwvi/volONRxiGL8cAy6NRa2jbPr3IVntXRpP+INiIf7CLB7q+IYEfp5bgrjafOQymwWVT8u3GTcv3phI3qVs4ltaL3ud+KCQKIKKRLB8WhwXKmJ28qi73SCufI55YPp/0yRtw+Wl8yQQsvyYCHn9t010CgYAsHhBIMYxFM+4pJjz/Xflv8d4cwDhFvIauydmCuBe2GOTpKqPFNF1yHlvlb8r2PENnJ660QD2snh1aRNAZTadzGx3lw5fwkhQLb/l6XOxfh53Kyx9UPR3r9dDgVXDLjdYN8moi++O9TUjzBZpwaxB4T6AIOssbQ1cG/pH4FcSFTQKBgCbqnSWORX1XLCbLGLS4DPC0kBj/sTRa+weemHZ+P6kIIzDTBgd5ORsxbq3ea7sO47cRFCWdNCxHGwRer0pb9jeb6og5nm1NW0z4M90gIICDWQgBeoS2Cj+S00hnQfSFR4RN3V7yI0Oitjbmah9mtw55bMBhuncqh3agrwxyqLCx" + ], + "certificate": [ + "MIICnTCCAYUCBgGVhuGg5jANBgkqhkiG9w0BAQsFADASMRAwDgYDVQQDDAdkZWZhdWx0MB4XDTI1MDMxMTIwMjMwOFoXDTM1MDMxMTIwMjQ0OFowEjEQMA4GA1UEAwwHZGVmYXVsdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKzDcb2wlwi2QYqpzRHGfl6RiVbyl5xlTr3Oj+Ak+CAw6o7klFm6RgBAj4soGI5VA0ZDcOamNGfBaKOINP8ATgbxDKLZARFvCYk/TQ9SWQBXKpVZLEjHRPWL5q2UqLXkrOcEih8b4dPdN3CxLUVJsJatQFbehLsfdIpnuYrgWy5sLJJPPNklHMYEbA6eIqHKwWffW5eqKtdA/u4eicjUtJO34faO6uReH7KhDY176DwEkZNUdEN6ST0+1Iof8HrZuIr/Uvo6pqBXHGIwZ2psu1PzURdl2rky69F4/0tlkT2/ngxefaJz8s9dwq/42QuWM2iqoxRgUXlvnN2CeREw3VkCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEADI8SCMfX98Y9D1/62CEEIcM6Ws5y8tCcQz+iWefjB+vWFOfzwHd3Fx64P4sC6Orl+hlYWbU3Qn2H+8F1FD4PmsEKDLRZMzSyXqTH5q4Z7UMSLUQe3s5jUfDmHesLRVFf1Qtj2sOCvZCm8NFjwbPBMK6qtzjrLa6Js7jwWFbh0p9ktxpyvxYxLW7KNxglBgIOqYmseKnYwxYKSsYIEcV/ONnGi0wed2xF2EpjGSqhXDmLZOAQhsjuUneSICTkmK83bZrHWa+v9SHvi3Ypo5tnInE3jbuitS/8CUwke5e27mw0wce5CL/Be65Iv/k80gd8tcPucIMga7c0pqFbsQnlSA==" + ], + "priority": [ + "100" + ], + "algorithm": [ + "RSA-OAEP" + ] + } + }, + { + "id": "a4d499fc-aced-40b4-9102-255159578eec", + "name": "hmac-generated-hs512", + "providerId": "hmac-generated", + "subComponents": {}, + "config": { + "kid": [ + "097e5e0b-731e-4051-8125-e6f2a95d85dd" + ], + "secret": [ + "C5PmVP-PUiEhOvaR8pO91VirtTrFqejyQGJRJDiauQLTiK7M064pqVxeRhuTuPnlJwmykGD0LVHP1Hfk315rt9zR-cyBL3JaVgZsfeU4jHwcwJixFWtmXN0XjG5Ql-UpdMUoVWucvI_TUMNffWuSI2beLHU3ik2NkMUjAXwQ_eM" + ], + "priority": [ + "100" + ], + "algorithm": [ + "HS512" + ] + } + }, + { + "id": "f5dc3169-59a8-4790-a9e5-e0bf3c78805c", + "name": "aes-generated", + "providerId": "aes-generated", + "subComponents": {}, + "config": { + "kid": [ + "a6afd833-da03-4f08-a7f7-c4b8eb2a375f" + ], + "secret": [ + "aOC5gC8wSuQzdQfwtHqH1w" + ], + "priority": [ + "100" + ] + } + }, + { + "id": "762fc122-3966-4253-8ff2-681aef52798a", + "name": "rsa-generated", + "providerId": "rsa-generated", + "subComponents": {}, + "config": { + "privateKey": [ + "MIIEpAIBAAKCAQEAsXBlZ1S+NSOWhZ9nGebBTh7TjGPxl0LbcFgKnB8uLnQa7VSAWjmicCB+3bVXIP29l0MtstKJx6tZhGaiIazHGgjtK44hagr0NJwf+B+LwilmzgwcDyrOY4NUMkoFDkoaf4fTN75UKTW+ELKlIkm85mu7X1WySeQg23QvX3tCzPaUoUBP6HnWQwdcc0LeuPRd3rp2LAKPDug2ZWP7VzAWsLbimykd+C8BSJzTYDC0hn2B5t10VHhzOPgpZRbcHPY/xOLKN31oAymLfPH+OxAe7Px0mGZFuH4O6CcQEErsqIFRjsfByMBofjmQJbgOcse9EHLdPZh4vp3GBIdzfwyedQIDAQABAoIBACxBlmkxb8edOrvVSEfjkC9F7QnFG8rYeBcLPoo2FLStbNHpE6JtYaCJ2lq+Eh8+an1B2jIR5R+SE1+8oY+4omhR9aW5a4ghd0tv7WFbrOYeoW+fAZie2p9jcCqi36Pyw60vWXU72Y31w5QympF0xtLJ6BAdNbzMU42o6u/rtHueUpxs9EtckxNvIAKCGcfkHTtpZ7fZkQdv8umsz1IKHyRWlIwdWDCnYRAw5TdjAwRBGk2epzOh9NDxbPiy28MP4JpmRYfmpB6uc33Q5y3krZhfL9TctA3UOPCVcWyVFbAeG4wEcQlB+g6Hqbs5R/krgii3AmdOhB7mlIzhSEQ4joECgYEA19EMUHlI2z275AABP7Mh63njM83UYN6dKnvsxbSCE3mR+0WQHe57Huqt3sr+WsYLre54h25zfw2TbQQjKrJ6uOINfhjwJX0FDkrlw1BO+GRJpzt8kOYskyB8o2arCxakkmhHFyazIKEk1DTh9jLFLXKSaruNJJw9iEQvzX+hlJECgYEA0noSmLjxS3UFANWzoY9ch+N/6fyvJ0Aojgg6lahLlAtL+v59tp4dulfjiCuyzGd0g1WgdPWpnCOHGR+FhikeKMna98PGrIfLL3rV3qjKjioHdtFUsK26pyYxBIu0FHtIKERqeT3tx+cTylXWA2nVd4auABTWzz3NuPOmyGXUjaUCgYEApVZiOMSyLER/TY0zZ7m0otIeXfGyYwQpJAMMweooPQNF81q3rjal3GmuCqE5fBF9oSKw9BCKKywbZcllp7BUlI+aBqDUWeQNm4WFwLwlw+YRBy1roRa1z4Fz+zsMjtIqAoAg9nuPf8/0hx58fkEnDkpYIazN1N5dxad3d9fv0gECgYEApgxGVZQ6UNReAR2XHJNUZaRWSsvdhvK3y+20AlOGZKJQ7BAQL50oSNWDnO8UnOvVYLOR5hPVHmhs8aYLmh8gOv+crzEVsRFke+3FgmbZfjSsNNHKpaQ5iBq6OyLYC/yCnbnz4fi4eafU1iDHuWOqVCS9azUFjvPsM8iNQLYNbT0CgYBqg900PIveL5qsmgVopmg+kimU/w+S/gCV+Gy1aNdzrMAIyrwT0MV7jsYf7eg5kirvtbgNqobjZERVaDdOkbHOW69lE975NoranH3HzYK13Rv3xktnYu5JNYXCEVTV+maG1K0V5BDy5zxleGa7GqVONesNDN0nOh9p+U0WHnpLTA==" + ], + "certificate": [ + "MIICnTCCAYUCBgGVhuGkdTANBgkqhkiG9w0BAQsFADASMRAwDgYDVQQDDAdkZWZhdWx0MB4XDTI1MDMxMTIwMjMwOVoXDTM1MDMxMTIwMjQ0OVowEjEQMA4GA1UEAwwHZGVmYXVsdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALFwZWdUvjUjloWfZxnmwU4e04xj8ZdC23BYCpwfLi50Gu1UgFo5onAgft21VyD9vZdDLbLSicerWYRmoiGsxxoI7SuOIWoK9DScH/gfi8IpZs4MHA8qzmODVDJKBQ5KGn+H0ze+VCk1vhCypSJJvOZru19VsknkINt0L197Qsz2lKFAT+h51kMHXHNC3rj0Xd66diwCjw7oNmVj+1cwFrC24pspHfgvAUic02AwtIZ9gebddFR4czj4KWUW3Bz2P8Tiyjd9aAMpi3zx/jsQHuz8dJhmRbh+DugnEBBK7KiBUY7HwcjAaH45kCW4DnLHvRBy3T2YeL6dxgSHc38MnnUCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAP8LumA7qBduayFtk67nbjI+sg7C6+auJJhclqXVP7qV2C3PpS4UXl1Hs4nuM6blhFMLZJWD+9FN7llsRw8HQg6pXg581QN/2pkYzr1uAP46/EyONYGbh0LkkLhKYHksJOABhkj6W7jRQ9/+1OMveREkbUMjlOOefdEoa6+zrT4sJPQAwNTXHcQrpjHQXEBysrbxO4TMqfdT1athivAMBAVikDVEI2uerarJ3CcM6tEFy4G3qdIEdXuIsx1bm0yPAri5WRIT4mhAFlOon1B6qqaYl6cdvw7L2s3qrCmMwkQbm8Kr7+7OleaCppLapi8Kt5v5XGjYV2y32Qj/zqDW2wA==" + ], + "priority": [ + "100" + ] + } + } + ] + }, + "internationalizationEnabled": false, + "supportedLocales": [], + "authenticationFlows": [ + { + "id": "ee04c896-46c6-46e5-8f5f-2c3f5f5e982f", + "alias": "Account verification options", + "description": "Method with which to verity the existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-email-verification", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "Verify Existing Account by Re-authentication", + "userSetupAllowed": false + } + ] + }, + { + "id": "78e019d7-5a2f-44f3-81d0-6fa566f21a32", + "alias": "Browser - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "3482050b-0ea3-4aee-8200-bcc78dfda38d", + "alias": "Direct Grant - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "direct-grant-validate-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "64987dfc-4f74-4abe-a9fa-47028f056258", + "alias": "First broker login - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "32144fde-d5b5-4a21-bdd5-fb8d92f3c435", + "alias": "Handle Existing Account", + "description": "Handle what to do if there is existing account with same email/username like authenticated identity provider", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-confirm-link", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "Account verification options", + "userSetupAllowed": false + } + ] + }, + { + "id": "53e1602d-6fb0-4dcb-81a8-e0e549165a2c", + "alias": "Reset - Conditional OTP", + "description": "Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "reset-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "4819cc94-f5e5-47ca-99ca-35abb57f80e4", + "alias": "User creation or linking", + "description": "Flow for the existing/non-existing user alternatives", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "create unique user config", + "authenticator": "idp-create-user-if-unique", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "Handle Existing Account", + "userSetupAllowed": false + } + ] + }, + { + "id": "5942a1e4-f029-40cd-ae92-830898bf97dc", + "alias": "Verify Existing Account by Re-authentication", + "description": "Reauthentication of existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "First broker login - Conditional OTP", + "userSetupAllowed": false + } + ] + }, + { + "id": "92632d82-47d5-4dd6-8a7e-8bf3e0ecd02f", + "alias": "browser", + "description": "browser based authentication", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-cookie", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorConfig": "oidc", + "authenticator": "identity-provider-redirector", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 25, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 30, + "autheticatorFlow": true, + "flowAlias": "forms", + "userSetupAllowed": false + } + ] + }, + { + "id": "dfedc4d5-a50e-4622-bba6-7e5ee9b2065c", + "alias": "clients", + "description": "Base authentication for clients", + "providerId": "client-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "client-secret", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "client-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "client-secret-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 30, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "client-x509", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 40, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "25ac344e-2d5e-4c5c-8f86-2b8ba2e3dfde", + "alias": "direct grant", + "description": "OpenID Connect Resource Owner Grant", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "direct-grant-validate-username", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "direct-grant-validate-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 30, + "autheticatorFlow": true, + "flowAlias": "Direct Grant - Conditional OTP", + "userSetupAllowed": false + } + ] + }, + { + "id": "fc07ca16-4d7d-4132-a6d6-352a566d19da", + "alias": "docker auth", + "description": "Used by Docker clients to authenticate against the IDP", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "docker-http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "eb9f31b1-f136-47bc-93d1-9fc8c62de3c5", + "alias": "first broker login", + "description": "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "review profile config", + "authenticator": "idp-review-profile", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "User creation or linking", + "userSetupAllowed": false + } + ] + }, + { + "id": "92f6f1d6-7684-4c3a-8623-1a1643cf1b07", + "alias": "forms", + "description": "Username, password, otp and other auth forms.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "Browser - Conditional OTP", + "userSetupAllowed": false + } + ] + }, + { + "id": "4701b039-0e51-435c-a867-76e602446315", + "alias": "registration", + "description": "registration flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-page-form", + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": true, + "flowAlias": "registration form", + "userSetupAllowed": false + } + ] + }, + { + "id": "4857befc-6fac-45f1-ae13-02830ab04646", + "alias": "registration form", + "description": "registration form", + "providerId": "form-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-user-creation", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "registration-password-action", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 50, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "registration-recaptcha-action", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 60, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "registration-terms-and-conditions", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 70, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "72627447-0fbe-4017-96ab-b12eacc75a2d", + "alias": "reset credentials", + "description": "Reset credentials for a user if they forgot their password or something", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "reset-credentials-choose-user", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "reset-credential-email", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "reset-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 30, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 40, + "autheticatorFlow": true, + "flowAlias": "Reset - Conditional OTP", + "userSetupAllowed": false + } + ] + }, + { + "id": "d8bec40b-7015-40c9-b069-c1bb1b1cf09e", + "alias": "saml ecp", + "description": "SAML ECP Profile Authentication Flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + } + ], + "authenticatorConfig": [ + { + "id": "6ee3b9e1-c19b-43f5-bc30-4dde2953aa3f", + "alias": "create unique user config", + "config": { + "require.password.update.after.registration": "false" + } + }, + { + "id": "420707ab-984e-4a95-a679-26cddf145323", + "alias": "oidc", + "config": { + "default.reference.maxAge": "10000", + "default.reference.value": "CILogon", + "defaultProvider": "oidc" + } + }, + { + "id": "6bab8c26-0c25-4cf3-a5a7-634cdfcf9f41", + "alias": "review profile config", + "config": { + "update.profile.on.first.login": "missing" + } + } + ], + "requiredActions": [ + { + "alias": "CONFIGURE_TOTP", + "name": "Configure OTP", + "providerId": "CONFIGURE_TOTP", + "enabled": true, + "defaultAction": false, + "priority": 10, + "config": {} + }, + { + "alias": "TERMS_AND_CONDITIONS", + "name": "Terms and Conditions", + "providerId": "TERMS_AND_CONDITIONS", + "enabled": false, + "defaultAction": false, + "priority": 20, + "config": {} + }, + { + "alias": "UPDATE_PASSWORD", + "name": "Update Password", + "providerId": "UPDATE_PASSWORD", + "enabled": true, + "defaultAction": false, + "priority": 30, + "config": {} + }, + { + "alias": "UPDATE_PROFILE", + "name": "Update Profile", + "providerId": "UPDATE_PROFILE", + "enabled": true, + "defaultAction": false, + "priority": 40, + "config": {} + }, + { + "alias": "VERIFY_EMAIL", + "name": "Verify Email", + "providerId": "VERIFY_EMAIL", + "enabled": true, + "defaultAction": false, + "priority": 50, + "config": {} + }, + { + "alias": "delete_account", + "name": "Delete Account", + "providerId": "delete_account", + "enabled": false, + "defaultAction": false, + "priority": 60, + "config": {} + }, + { + "alias": "webauthn-register", + "name": "Webauthn Register", + "providerId": "webauthn-register", + "enabled": true, + "defaultAction": false, + "priority": 70, + "config": {} + }, + { + "alias": "webauthn-register-passwordless", + "name": "Webauthn Register Passwordless", + "providerId": "webauthn-register-passwordless", + "enabled": true, + "defaultAction": false, + "priority": 80, + "config": {} + }, + { + "alias": "VERIFY_PROFILE", + "name": "Verify Profile", + "providerId": "VERIFY_PROFILE", + "enabled": true, + "defaultAction": false, + "priority": 90, + "config": {} + }, + { + "alias": "update_user_locale", + "name": "Update User Locale", + "providerId": "update_user_locale", + "enabled": true, + "defaultAction": false, + "priority": 1000, + "config": {} + } + ], + "browserFlow": "browser", + "registrationFlow": "registration", + "directGrantFlow": "direct grant", + "resetCredentialsFlow": "reset credentials", + "clientAuthenticationFlow": "clients", + "dockerAuthenticationFlow": "docker auth", + "firstBrokerLoginFlow": "first broker login", + "attributes": { + "cibaBackchannelTokenDeliveryMode": "poll", + "cibaAuthRequestedUserHint": "login_hint", + "clientOfflineSessionMaxLifespan": "0", + "oauth2DevicePollingInterval": "5", + "clientSessionIdleTimeout": "0", + "actionTokenGeneratedByUserLifespan.idp-verify-account-via-email": "", + "actionTokenGeneratedByUserLifespan.verify-email": "", + "clientOfflineSessionIdleTimeout": "0", + "actionTokenGeneratedByUserLifespan.execute-actions": "", + "cibaInterval": "5", + "realmReusableOtpCode": "false", + "cibaExpiresIn": "120", + "oauth2DeviceCodeLifespan": "600", + "parRequestUriLifespan": "60", + "clientSessionMaxLifespan": "0", + "frontendUrl": "", + "acr.loa.map": "{}", + "shortVerificationUri": "", + "actionTokenGeneratedByUserLifespan.reset-credentials": "" + }, + "keycloakVersion": "24.0.0", + "userManagedAccessAllowed": false, + "clientProfiles": { + "profiles": [] + }, + "clientPolicies": { + "policies": [] + } +} \ No newline at end of file diff --git a/dev-tools/ansible/roles/keycloak/templates/vhost.conf.j2 b/dev-tools/ansible/roles/keycloak/templates/vhost.conf.j2 index b03a0dcf0a..b5320137e3 100644 --- a/dev-tools/ansible/roles/keycloak/templates/vhost.conf.j2 +++ b/dev-tools/ansible/roles/keycloak/templates/vhost.conf.j2 @@ -32,14 +32,22 @@ ServerName {{ keycloak_vhost_servername }} RequestHeader set X-Forwarded-Proto "https" - ProxyPass / "http://localhost:8080/" - ProxyPassReverse / "http://localhost:8080/" + RequestHeader set X-Forwarded-Port "443" + ProxyPass / "https://localhost:8443/" + ProxyPassReverse / "https://localhost:8443/" ProxyPreserveHost On + SSLProxyEngine On + SSLProxyVerify none # See https://issues.redhat.com/browse/KEYCLOAK-3067 for more info LimitRequestFieldSize 65536 +{% if ansible_os_family == "Debian" %} + ErrorLog /var/log/apache2/keycloak.error.log + CustomLog /var/log/apache2/keycloak.requests.log combined +{% else %} ErrorLog /var/log/httpd/keycloak.error.log CustomLog /var/log/httpd/keycloak.requests.log combined +{% endif %} SSLEngine on # Disable SSLv3 which is vulnerable to the POODLE attack diff --git a/dev-tools/ansible/roles/letsencrypt/tasks/main.yml b/dev-tools/ansible/roles/letsencrypt/tasks/main.yml index 6fee7fb88d..16cb7d1f18 100644 --- a/dev-tools/ansible/roles/letsencrypt/tasks/main.yml +++ b/dev-tools/ansible/roles/letsencrypt/tasks/main.yml @@ -23,25 +23,61 @@ - include_tasks: install_deps_{{ ansible_distribution }}_{{ ansible_distribution_major_version }}.yml when: ansible_os_family == "RedHat" -- name: add Certbot PPA repository - apt_repository: - repo: "ppa:certbot/certbot" +# Note: Certbot PPA is deprecated for Ubuntu 24.04+. Install from Ubuntu repositories or snap instead. +# For Ubuntu 22.04 and earlier, we can still use the PPA. +# For Ubuntu 24.04+, certbot is available in the universe repository. +- name: Install software-properties-common for add-apt-repository (Ubuntu < 24.04) + apt: + name: software-properties-common + state: present + update_cache: yes become: yes - when: ansible_os_family == "Debian" + when: ansible_os_family == "Debian" and (ansible_distribution_major_version | int < 24) + +- name: add Certbot PPA repository (Ubuntu < 24.04) + shell: | + add-apt-repository -y ppa:certbot/certbot + apt-get update + args: + creates: /etc/apt/sources.list.d/certbot-ubuntu-certbot-*.list + become: yes + when: ansible_os_family == "Debian" and (ansible_distribution_major_version | int < 24) + +- name: Find and remove deprecated certbot PPA files (Ubuntu 24.04+) + shell: | + rm -f /etc/apt/sources.list.d/certbot-ubuntu-certbot-*.list /etc/apt/sources.list.d/certbot-ubuntu-certbot-*.list.save /etc/apt/sources.list.d/certbot-ubuntu-certbot-*.sources 2>/dev/null || true + become: yes + when: ansible_os_family == "Debian" and (ansible_distribution_major_version | int >= 24) + ignore_errors: yes + +- name: Enable universe repository for certbot (Ubuntu 24.04+) + shell: | + add-apt-repository -y universe || true + apt-get update -o APT::Get::AllowUnauthenticated=true || apt-get update --allow-insecure-repositories || true + become: yes + when: ansible_os_family == "Debian" and (ansible_distribution_major_version | int >= 24) + ignore_errors: yes + +- name: Install Certbot and dependencies (Ubuntu 24.04+) + shell: | + apt-get install -y certbot python3-certbot-apache || apt-get install -y certbot python3-certbot-apache --allow-unauthenticated + become: yes + when: ansible_os_family == "Debian" and (ansible_distribution_major_version | int >= 24) -- name: Install Certbot and dependencies (Debian) +- name: Install Certbot and dependencies (Ubuntu < 24.04) apt: name={{ item }} state=latest update_cache=yes with_items: - certbot - python-certbot-apache become: yes - when: ansible_os_family == "Debian" + when: ansible_os_family == "Debian" and (ansible_distribution_major_version | int < 24) -# Note: on Ubuntu crontab is automatically created to run cert renewal. Only -# CentOS requires enabling the certbot-renew timer. +# Note: Ubuntu automatically sets up certificate renewal via systemd timer. +# The timer is created when certbot is installed via apt. +# CentOS/Rocky require manual enabling of the certbot-renew timer. - name: enable certbot (letsencrypt) renewal systemd: diff --git a/dev-tools/ansible/roles/rabbitmq/defaults/main.yml b/dev-tools/ansible/roles/rabbitmq/defaults/main.yml new file mode 100644 index 0000000000..dcad19a193 --- /dev/null +++ b/dev-tools/ansible/roles/rabbitmq/defaults/main.yml @@ -0,0 +1,23 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +# RabbitMQ Erlang version (compatible with latest RabbitMQ 3.x) +rabbitmq_erlang_version: "26.3.2.3" diff --git a/dev-tools/ansible/roles/rabbitmq/tasks/main.yml b/dev-tools/ansible/roles/rabbitmq/tasks/main.yml index b9a3ed43c5..4d26cd5ac6 100644 --- a/dev-tools/ansible/roles/rabbitmq/tasks/main.yml +++ b/dev-tools/ansible/roles/rabbitmq/tasks/main.yml @@ -21,25 +21,54 @@ --- ################################################################################ # Setup and run rabbitmq -- name: Install erlang latest version (CentOS) - yum: name=https://www.rabbitmq.com/releases/erlang/erlang-18.3-1.el7.centos.x86_64.rpm state=present + +# Ubuntu/Debian – install RabbitMQ and Erlang from distro repositories +- name: Install RabbitMQ prerequisites (Ubuntu/Debian) + apt: + name: + - erlang + - rabbitmq-server + state: present + update_cache: yes + become: yes + when: ansible_os_family == "Debian" + +# Rocky/CentOS: Add RabbitMQ RPM repository +- name: Install erlang from RabbitMQ repository (Rocky/CentOS) + yum: + name: "https://github.com/rabbitmq/erlang-rpm/releases/download/v{{ rabbitmq_erlang_version }}/erlang-{{ rabbitmq_erlang_version }}-1.el{{ ansible_distribution_major_version }}.noarch.rpm" + state: present become: yes - when: ansible_distribution == "CentOS" + when: ansible_os_family == "RedHat" + ignore_errors: yes -- name: Install erlang latest version (Rocky) - dnf: name=https://github.com/rabbitmq/erlang-rpm/releases/download/v24.1.4/erlang-24.1.4-1.el8.x86_64.rpm +- name: Add RabbitMQ YUM repository (Rocky/CentOS) + yum_repository: + name: rabbitmq_server + description: RabbitMQ Server + baseurl: "https://packagecloud.io/rabbitmq/rabbitmq-server/el/{{ ansible_distribution_major_version }}/$basearch" + gpgcheck: yes + gpgkey: "https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey" + state: present become: yes - when: ansible_distribution == "Rocky" + when: ansible_os_family == "RedHat" -- name: Install Rabbitmq rpm (CentOS) - yum: name=https://www.rabbitmq.com/releases/rabbitmq-server/v3.6.3/rabbitmq-server-3.6.3-1.noarch.rpm state=present +# Install RabbitMQ server +- name: Install RabbitMQ server (Ubuntu) + apt: + name: rabbitmq-server + state: present + update_cache: yes become: yes - when: ansible_distribution == "CentOS" + when: ansible_os_family == "Debian" -- name: Install Rabbitmq rpm (Rocky) - dnf: name=https://github.com/rabbitmq/rabbitmq-server/releases/download/v3.10.6/rabbitmq-server-3.10.6-1.el8.noarch.rpm +- name: Install RabbitMQ server (Rocky/CentOS) + yum: + name: rabbitmq-server + state: present + update_cache: yes become: yes - when: ansible_distribution == "Rocky" + when: ansible_os_family == "RedHat" - name: allow only selected networks to access Airavata RabbitMQ firewalld: diff --git a/dev-tools/ansible/roles/reverse_proxy/defaults/main.yml b/dev-tools/ansible/roles/reverse_proxy/defaults/main.yml new file mode 100644 index 0000000000..be66aa421f --- /dev/null +++ b/dev-tools/ansible/roles/reverse_proxy/defaults/main.yml @@ -0,0 +1,39 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +# Reverse proxy configuration defaults +reverse_proxy_service_name: + CentOS_7: httpd + Rocky_8: httpd + Ubuntu_22: apache2 + Ubuntu_24: apache2 + default: apache2 + +reverse_proxy_config_dir: + CentOS_7: /etc/httpd + Rocky_8: /etc/httpd + Ubuntu_22: /etc/apache2 + Ubuntu_24: /etc/apache2 + default: /etc/apache2 + +reverse_proxy_ssl_dir: "/etc/apache2/ssl-available" +reverse_proxy_vhost_dir: "/etc/apache2/sites-available" + diff --git a/dev-tools/ansible/roles/reverse_proxy/handlers/main.yml b/dev-tools/ansible/roles/reverse_proxy/handlers/main.yml new file mode 100644 index 0000000000..05c6f8e103 --- /dev/null +++ b/dev-tools/ansible/roles/reverse_proxy/handlers/main.yml @@ -0,0 +1,33 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +- name: reload reverse proxy + service: + name: "{{ reverse_proxy_service_name[ansible_distribution + '_' + ansible_distribution_major_version] }}" + state: reloaded + become: yes + +- name: restart reverse proxy + service: + name: "{{ reverse_proxy_service_name[ansible_distribution + '_' + ansible_distribution_major_version] }}" + state: restarted + become: yes + diff --git a/dev-tools/ansible/roles/reverse_proxy/tasks/main.yml b/dev-tools/ansible/roles/reverse_proxy/tasks/main.yml new file mode 100644 index 0000000000..5e048a3a10 --- /dev/null +++ b/dev-tools/ansible/roles/reverse_proxy/tasks/main.yml @@ -0,0 +1,68 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +# Install Apache2/httpd +- name: Install Apache2 (Ubuntu/Debian) + apt: + name: apache2 + state: present + update_cache: yes + become: yes + when: ansible_os_family == "Debian" + +- name: Install httpd (Rocky/CentOS) + yum: + name: httpd + state: present + update_cache: yes + become: yes + when: ansible_distribution == "CentOS" + +- name: Install httpd (Rocky) + dnf: + name: httpd + state: present + update_cache: yes + become: yes + when: ansible_distribution == "Rocky" + +# Enable required modules (Ubuntu/Debian) +- name: Enable Apache2 modules (Ubuntu/Debian) + shell: a2enmod {{ item }} + args: + creates: /etc/apache2/mods-enabled/{{ item }}.load + become: yes + when: ansible_os_family == "Debian" + loop: + - proxy + - proxy_http + - ssl + - rewrite + - headers + +# Start and enable service +- name: Start and enable Apache2/httpd + service: + name: "{{ reverse_proxy_service_name[ansible_distribution + '_' + ansible_distribution_major_version] | default(reverse_proxy_service_name[ansible_distribution] | default(reverse_proxy_service_name['default'])) }}" + state: started + enabled: yes + become: yes + diff --git a/dev-tools/ansible/roles/zookeeper/tasks/main.yml b/dev-tools/ansible/roles/zookeeper/tasks/main.yml index 2cf508550d..9ec0dd0dda 100644 --- a/dev-tools/ansible/roles/zookeeper/tasks/main.yml +++ b/dev-tools/ansible/roles/zookeeper/tasks/main.yml @@ -46,12 +46,6 @@ - restart zookeeper become: yes -- name: Copy java.env file - template: src=java.env.j2 dest="{{ zookeeper_dir }}/conf/java.env" owner="{{ user }}" group="{{ group }}" mode="u=rw,g=r,o=r" - notify: - - restart zookeeper - become: yes - - name: Check if systemd exists stat: path=/usr/lib/systemd/system/ register: systemd_check @@ -67,4 +61,12 @@ - name: reload systemd daemons command: systemctl daemon-reload become: yes + +- name: Start and enable Zookeeper service + service: + name: zookeeper + state: started + enabled: yes + become: yes + when: systemd_check.stat.exists == true ... diff --git a/dev-tools/ansible/roles/zookeeper/templates/java.env.j2 b/dev-tools/ansible/roles/zookeeper/templates/java.env.j2 deleted file mode 100644 index 356192c818..0000000000 --- a/dev-tools/ansible/roles/zookeeper/templates/java.env.j2 +++ /dev/null @@ -1 +0,0 @@ -export ZOO_LOG_DIR={{ zookeeper_dir }}/logs \ No newline at end of file diff --git a/dev-tools/ansible/roles/zookeeper/templates/zoo.cfg.j2 b/dev-tools/ansible/roles/zookeeper/templates/zoo.cfg.j2 index 0c40776b92..0213446228 100644 --- a/dev-tools/ansible/roles/zookeeper/templates/zoo.cfg.j2 +++ b/dev-tools/ansible/roles/zookeeper/templates/zoo.cfg.j2 @@ -12,6 +12,8 @@ syncLimit={{sync_limit}} dataDir={{zookeeper_data_dir}} # the port at which the clients will connect clientPort={{ client_port }} +# AdminServer port (default is 8080, changed to avoid conflicts with Keycloak) +admin.serverPort={{ zookeeper_admin_server_port }} # the maximum number of client connections. # increase this if you need to handle more clients #maxClientCnxns=60 diff --git a/dev-tools/ansible/roles/zookeeper/vars/main.yml b/dev-tools/ansible/roles/zookeeper/vars/main.yml index 68eada2526..f9b7cb6825 100644 --- a/dev-tools/ansible/roles/zookeeper/vars/main.yml +++ b/dev-tools/ansible/roles/zookeeper/vars/main.yml @@ -22,7 +22,7 @@ #Variables associated with this role # zookeeper related variable zookeeper_version: 3.8.4 -zookeeper_url: http://archive.apache.org/dist/zookeeper/zookeeper-{{zookeeper_version}}/zookeeper-{{zookeeper_version}}.tar.gz +zookeeper_url: http://archive.apache.org/dist/zookeeper/zookeeper-{{zookeeper_version}}/apache-zookeeper-{{zookeeper_version}}-bin.tar.gz apt_cache_timeout: 3600 client_port: "{{ zookeeper_client_port }}" @@ -31,7 +31,9 @@ sync_limit: 2 tick_time: 2000 data_dir: /var/lib/zookeeper log_dir: /var/log/zookeeper -zookeeper_dir: "{{ user_home }}/zookeeper-{{zookeeper_version}}" +zookeeper_dir: "{{ user_home }}/apache-zookeeper-{{zookeeper_version}}-bin" zookeeper_data_dir: "{{ zookeeper_dir }}/data" +# AdminServer port (default is 8080, but conflicts with Keycloak) +zookeeper_admin_server_port: 8081 ... diff --git a/dev-tools/ansible/start_services.yml b/dev-tools/ansible/start_services.yml new file mode 100644 index 0000000000..4c3bc2358d --- /dev/null +++ b/dev-tools/ansible/start_services.yml @@ -0,0 +1,47 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +# Start Airavata Services Playbook +# +# This playbook starts all Airavata services. +# +# Usage: +# ansible-playbook -i inventories/ start_services.yml --ask-vault-pass + +- name: Start Airavata Services + hosts: airavata_servers + become: yes + become_user: "{{ deploy_user | default(ansible_user) }}" + + tasks: + - name: Display start information + debug: + msg: "Starting Airavata services on {{ inventory_hostname }}" + + - name: Start all services + include_role: + name: airavata_services + tasks_from: start_services + + - name: Display completion message + debug: + msg: "All Airavata services started on {{ inventory_hostname }}" + diff --git a/dev-tools/ansible/stop_services.yml b/dev-tools/ansible/stop_services.yml new file mode 100644 index 0000000000..af22452233 --- /dev/null +++ b/dev-tools/ansible/stop_services.yml @@ -0,0 +1,47 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +# Stop Airavata Services Playbook +# +# This playbook stops all Airavata services and verifies ports are free. +# +# Usage: +# ansible-playbook -i inventories/ stop_services.yml --ask-vault-pass + +- name: Stop Airavata Services + hosts: airavata_servers + become: yes + become_user: "{{ deploy_user | default(ansible_user) }}" + + tasks: + - name: Display stop information + debug: + msg: "Stopping Airavata services on {{ inventory_hostname }}" + + - name: Stop all services and verify ports are free + include_role: + name: airavata_services + tasks_from: stop_services + + - name: Display completion message + debug: + msg: "All Airavata services stopped and ports verified on {{ inventory_hostname }}" +