From afe1e50e5f6fae219a4dce309d53f59efa149192 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Wed, 13 Aug 2025 12:35:52 +0530 Subject: [PATCH 01/25] only keep control machine and couch machine --- environments/backup-production/terraform.yml | 279 ------------------- 1 file changed, 279 deletions(-) diff --git a/environments/backup-production/terraform.yml b/environments/backup-production/terraform.yml index 12c3e48959..6519ffda43 100644 --- a/environments/backup-production/terraform.yml +++ b/environments/backup-production/terraform.yml @@ -24,108 +24,6 @@ servers: group: "control" os: jammy - - server_name: "djangomanage_a0-bk-production" - server_instance_type: t3a.large - network_tier: "app-private" - az: "c" - volume_size: 20 - group: "django_manage" - os: jammy - - - server_name: "web_a{i}-bk-production" - server_instance_type: t3a.large - network_tier: "app-private" - az: "c" - volume_size: 20 - volume_encrypted: yes - group: "hq_webworkers" - os: jammy - count: 2 - - - server_name: "web_b{i}-bk-production" - server_instance_type: t3a.large - network_tier: "app-private" - az: "b" - volume_size: 20 - group: "mobile_webworkers" - os: jammy - count: 2 - server_auto_recovery: true - - - server_name: "esmaster_a1-bk-production" - server_instance_type: t3a.xlarge - network_tier: "db-private" - az: "c" - group: "elasticsearch" - os: jammy - volume_size: 20 - - - server_name: "esmaster_b1-bk-production" - server_instance_type: t3a.xlarge - network_tier: "db-private" - az: "b" - group: "elasticsearch" - os: jammy - volume_size: 20 - volume_encrypted: yes - - - server_name: "esmaster_c1-bk-production" - server_instance_type: t3a.xlarge - network_tier: "db-private" - az: "c" - group: "elasticsearch" - os: jammy - volume_size: 20 - volume_encrypted: yes - - - server_name: "es_a2{i}-bk-production" - server_instance_type: t3a.xlarge - network_tier: "db-private" - az: "c" - volume_size: 20 - volume_encrypted: yes - block_device: - volume_size: 20 - group: "elasticsearch" - os: jammy - count: 4 - - - server_name: "es_b2{i}-bk-production" - server_instance_type: t3a.xlarge - network_tier: "db-private" - az: "b" - volume_size: 20 - volume_encrypted: yes - block_device: - volume_size: 2800 - group: "elasticsearch" - os: jammy - count: 4 - - - server_name: "es_a3{i}-bk-production" - server_instance_type: t3a.xlarge - network_tier: "db-private" - az: "c" - volume_size: 20 - volume_encrypted: yes - block_device: - volume_size: 20 - group: "elasticsearch" - os: jammy - count: 3 - - - server_name: "es_b3{i}-bk-production" - server_instance_type: t3a.xlarge - network_tier: "db-private" - az: "b" - volume_size: 20 - volume_encrypted: yes - block_device: - volume_size: 2800 - group: "elasticsearch" - os: jammy - count: 3 - - server_name: "couch_a0-bk-production" server_instance_type: t3a.small network_tier: "db-private" @@ -138,162 +36,6 @@ servers: group: "couchdb2" os: jammy - - server_name: "rabbit_a0-bk-production" - server_instance_type: t3a.small - network_tier: "db-private" - az: "c" - volume_size: 20 - volume_encrypted: yes - group: "rabbitmq" - os: jammy - - - server_name: "celery_a{i}-bk-production" - server_instance_type: t3a.medium - network_tier: "app-private" - az: "c" - volume_size: 20 - volume_encrypted: yes - group: "celery" - os: jammy - count: 1 - - - server_name: "pillow_a{i}-bk-production" - server_instance_type: t3a.medium - network_tier: "app-private" - az: "c" - volume_size: 20 - group: "pillowtop" - os: jammy - count: 1 - - - server_name: "formplayer_a{i}-bk-production" - server_instance_type: t3a.small - network_tier: "app-private" - az: "c" - volume_size: 20 - group: "formplayer" - os: jammy - count: 1 - - - server_name: "kafka_a0-bk-production" - server_instance_type: t3a.small - network_tier: "db-private" - az: "c" - volume_size: 20 - volume_encrypted: no - group: "kafka" - os: jammy - - - server_name: "pgbouncer_a0-bk-production" - server_instance_type: t3a.small - network_tier: "db-private" - az: "c" - volume_size: 20 - volume_encrypted: yes - group: "postgresql" - os: jammy - -proxy_servers: - - server_name: "proxy_a0-bk-production" - server_instance_type: t3a.large - network_tier: "public" - az: "c" - volume_size: 20 - group: "proxy" - os: jammy - -rds_instances: - - identifier: "pgmain0-bk-production" - instance_type: db.t3.micro - storage: 20 - multi_az: false - engine_version: "14" - monitoring_interval: 0 - - - identifier: "pgucr0-bk-production" - instance_type: db.t3.micro - storage: 20 - multi_az: false - engine_version: "14" - monitoring_interval: 0 - - - identifier: "pgshard1-bk-production" - instance_type: db.t3.micro - storage: 20 - multi_az: false - engine_version: "14" - monitoring_interval: 0 - - - identifier: "pgshard2-bk-production" - instance_type: db.t3.micro - storage: 20 - multi_az: false - engine_version: "14" - monitoring_interval: 0 - - - identifier: "pgshard3-bk-production" - instance_type: db.t3.micro - storage: 20 - multi_az: false - engine_version: "14" - monitoring_interval: 0 - - - identifier: "pgshard4-bk-production" - instance_type: db.t3.micro - storage: 20 - multi_az: false - engine_version: "14" - monitoring_interval: 0 - - - identifier: "pgshard5-bk-production" - instance_type: db.t3.micro - storage: 20 - multi_az: false - engine_version: "14" - monitoring_interval: 0 - - - identifier: "pgsynclog0-bk-production" - instance_type: db.t3.micro - storage: 20 - multi_az: false - engine_version: "14" - monitoring_interval: 0 - - - identifier: "pgformplayer0-bk-production" - instance_type: db.t3.micro - storage: 20 - multi_az: false - engine_version: "14" - monitoring_interval: 0 - - - identifier: "pgauditcare0-bk-production" - instance_type: db.t3.micro - storage: 20 - multi_az: false - engine_version: "14" - monitoring_interval: 0 - -pgbouncer_nlbs: - - name: pgformplayer_nlb-bk-production - targets: - - pgbouncer_a0-bk-production - - - name: pgmain_nlb-bk-production - targets: - - pgbouncer_a0-bk-production - - - name: pgucr_nlb-bk-production - targets: - - pgbouncer_a0-bk-production - - - name: pgsynclogs_nlb-bk-production - targets: - - pgbouncer_a0-bk-production - - - name: pgshard_nlb-bk-production - targets: - - pgbouncer_a0-bk-production - internal_albs: - name: "couch_alb-bk-production" listener_port: 25984 @@ -302,29 +44,8 @@ internal_albs: targets: - couch_a0-bk-production -elasticache_cluster: - create: yes - cache_node_type: "cache.t3.micro" - cache_engine_version: "7.0" - automatic_failover: true - transit_encryption: false - at_rest_encryption: true - auto_minor_version: false - cluster_size: 2 - maintenance_window: "sun:08:30-sun:09:30" - r53_private_zone: create: yes domain_name: "bk-production.commcare.local" create_record: yes route_names: "redis" - -efs_file_systems: - - efs_name: "efs_bk-production" - create: yes - transition_to_ia: "AFTER_14_DAYS" - create_access: no - create_mount: yes - create_record: yes - domain_name: "bk-production.commcare.local" - route_names: "shared-efs" From 0242d0fd65cf598fb8c3e3b9303e55f11c47904f Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Wed, 20 Aug 2025 17:28:20 +0530 Subject: [PATCH 02/25] update volume size for couch --- environments/backup-production/terraform.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/environments/backup-production/terraform.yml b/environments/backup-production/terraform.yml index 6519ffda43..ffdf954f74 100644 --- a/environments/backup-production/terraform.yml +++ b/environments/backup-production/terraform.yml @@ -31,7 +31,7 @@ servers: volume_size: 20 volume_encrypted: yes block_device: - volume_size: 20 + volume_size: 1000 encrypted: yes group: "couchdb2" os: jammy @@ -44,8 +44,3 @@ internal_albs: targets: - couch_a0-bk-production -r53_private_zone: - create: yes - domain_name: "bk-production.commcare.local" - create_record: yes - route_names: "redis" From b4907453d23acb77faa31ab18335c05039feaea3 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Fri, 22 Aug 2025 16:54:04 +0530 Subject: [PATCH 03/25] add 2 new couchdb nodes --- environments/backup-production/terraform.yml | 24 ++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/environments/backup-production/terraform.yml b/environments/backup-production/terraform.yml index ffdf954f74..4c03f28e44 100644 --- a/environments/backup-production/terraform.yml +++ b/environments/backup-production/terraform.yml @@ -36,6 +36,30 @@ servers: group: "couchdb2" os: jammy + - server_name: "couch_a1-bk-production" + server_instance_type: t3a.small + network_tier: "db-private" + az: "c" + volume_size: 20 + volume_encrypted: yes + block_device: + volume_size: 1000 + encrypted: yes + group: "couchdb2" + os: jammy + + - server_name: "couch_a2-bk-production" + server_instance_type: t3a.small + network_tier: "db-private" + az: "c" + volume_size: 20 + volume_encrypted: yes + block_device: + volume_size: 1000 + encrypted: yes + group: "couchdb2" + os: jammy + internal_albs: - name: "couch_alb-bk-production" listener_port: 25984 From 2ad36bd1d691e015bb0e4728f80cf3b11ccc7669 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Mon, 25 Aug 2025 22:16:02 +0530 Subject: [PATCH 04/25] update inventory, aws-resources accordingly --- .../backup-production/aws-resources.yml | 79 +---- environments/backup-production/inventory.ini | 309 +----------------- .../backup-production/inventory.ini.j2 | 220 +------------ environments/staging/inventory.ini | 222 ------------- 4 files changed, 16 insertions(+), 814 deletions(-) diff --git a/environments/backup-production/aws-resources.yml b/environments/backup-production/aws-resources.yml index bbca3326a6..340da16a0d 100644 --- a/environments/backup-production/aws-resources.yml +++ b/environments/backup-production/aws-resources.yml @@ -1,82 +1,11 @@ -celery_a000-bk-production: 10.212.11.179 -celery_a000-bk-production.instance_id: i-0ed9962ffe8e16c86 control_a0-bk-production: 10.212.11.9 control_a0-bk-production.instance_id: i-0fb549d5e815a6079 couch_a0-bk-production: 10.212.41.153 couch_a0-bk-production.instance_id: i-0113dcf847ca28abf +couch_a1-bk-production: 10.212.41.62 +couch_a1-bk-production.instance_id: i-0dde79ea855218d9f +couch_a2-bk-production: 10.212.41.29 +couch_a2-bk-production.instance_id: i-06d41c8d2726ca675 couch_alb-bk-production: internal-couch-alb-bk-production-62333703.us-east-2.elb.amazonaws.com -djangomanage_a0-bk-production: 10.212.11.191 -djangomanage_a0-bk-production.instance_id: i-0a7d4c21b7ee0f4a2 -efs_bk-production-efs: fs-062f504e792e38d2b.efs.us-east-2.amazonaws.com -es_a2000-bk-production: 10.212.41.16 -es_a2000-bk-production.instance_id: i-04f32d6467cfc8994 -es_a2001-bk-production: 10.212.41.84 -es_a2001-bk-production.instance_id: i-04a205c847e9bee36 -es_a2002-bk-production: 10.212.41.144 -es_a2002-bk-production.instance_id: i-061bc80b47d4af996 -es_a2003-bk-production: 10.212.41.223 -es_a2003-bk-production.instance_id: i-0c62dcc83cbef4abf -es_a3000-bk-production: 10.212.41.251 -es_a3000-bk-production.instance_id: i-03963aebbab9f388f -es_a3001-bk-production: 10.212.41.161 -es_a3001-bk-production.instance_id: i-02d56fd603e9ae729 -es_a3002-bk-production: 10.212.41.102 -es_a3002-bk-production.instance_id: i-079edad4c6f6e74ce -es_b2000-bk-production: 10.212.40.151 -es_b2000-bk-production.instance_id: i-0275feba1ef4a92fa -es_b2001-bk-production: 10.212.40.91 -es_b2001-bk-production.instance_id: i-07b8da671f6fd5bc5 -es_b2002-bk-production: 10.212.40.228 -es_b2002-bk-production.instance_id: i-09b01fed0945164ec -es_b2003-bk-production: 10.212.40.211 -es_b2003-bk-production.instance_id: i-0a7f6c9b5bd336322 -es_b3000-bk-production: 10.212.40.146 -es_b3000-bk-production.instance_id: i-064a0657515defb73 -es_b3001-bk-production: 10.212.40.104 -es_b3001-bk-production.instance_id: i-09a86e94b49b9f0a1 -es_b3002-bk-production: 10.212.40.172 -es_b3002-bk-production.instance_id: i-04e547f605e4b447d -esmaster_a1-bk-production: 10.212.41.133 -esmaster_a1-bk-production.instance_id: i-03450966b4641a16e -esmaster_b1-bk-production: 10.212.40.55 -esmaster_b1-bk-production.instance_id: i-04c5ebae4a1cd10ab -esmaster_c1-bk-production: 10.212.41.228 -esmaster_c1-bk-production.instance_id: i-0160322ab9ae0b0b1 formplayer-efs: fs-0287582bff16a9a0b.efs.us-east-2.amazonaws.com -formplayer_a000-bk-production: 10.212.11.163 -formplayer_a000-bk-production.instance_id: i-0adbcac28a4016ffc frontend_alb-bk-production: internal-frontend-alb-bk-production-194015126.us-east-2.elb.amazonaws.com -kafka_a0-bk-production: 10.212.41.30 -kafka_a0-bk-production.instance_id: i-03e12c2f481fd8af2 -pgauditcare0-bk-production: pgauditcare0-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com -pgbouncer_a0-bk-production: 10.212.41.150 -pgbouncer_a0-bk-production.instance_id: i-0582ef2a9c57fa780 -pgformplayer0-bk-production: pgformplayer0-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com -pgformplayer_nlb-bk-production: pgformplayer-nlb-bk-production-17249f03f744a864.elb.us-east-2.amazonaws.com -pgmain0-bk-production: pgmain0-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com -pgmain_nlb-bk-production: pgmain-nlb-bk-production-75f4dcf6a91cd6e1.elb.us-east-2.amazonaws.com -pgshard1-bk-production: pgshard1-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com -pgshard2-bk-production: pgshard2-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com -pgshard3-bk-production: pgshard3-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com -pgshard4-bk-production: pgshard4-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com -pgshard5-bk-production: pgshard5-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com -pgshard_nlb-bk-production: pgshard-nlb-bk-production-99be8739d5b85b57.elb.us-east-2.amazonaws.com -pgsynclog0-bk-production: pgsynclog0-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com -pgsynclogs_nlb-bk-production: pgsynclogs-nlb-bk-production-af78decde238eb6b.elb.us-east-2.amazonaws.com -pgucr0-bk-production: pgucr0-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com -pgucr_nlb-bk-production: pgucr-nlb-bk-production-c4336e567a87e30d.elb.us-east-2.amazonaws.com -pillow_a000-bk-production: 10.212.11.180 -pillow_a000-bk-production.instance_id: i-0b23885a5a4daf320 -proxy_a0-bk-production: 10.212.21.205 -proxy_a0-bk-production.instance_id: i-0c277b935dc317b2a -proxy_a0-bk-production.public_ip: 52.15.147.217 -rabbit_a0-bk-production: 10.212.41.209 -rabbit_a0-bk-production.instance_id: i-01640329a616749b0 -web_a000-bk-production: 10.212.11.103 -web_a000-bk-production.instance_id: i-09e2b15106899013a -web_a001-bk-production: 10.212.11.64 -web_a001-bk-production.instance_id: i-04e63e19d14c9cce9 -web_b000-bk-production: 10.212.10.147 -web_b000-bk-production.instance_id: i-085f4132f6ab1fcb7 -web_b001-bk-production: 10.212.10.211 -web_b001-bk-production.instance_id: i-04c20b4bab5c07be7 diff --git a/environments/backup-production/inventory.ini b/environments/backup-production/inventory.ini index 58868b4f85..1dd01749bf 100644 --- a/environments/backup-production/inventory.ini +++ b/environments/backup-production/inventory.ini @@ -1,300 +1,14 @@ -[proxy_a0] -10.212.21.205 hostname=proxy-a0-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0c277b935dc317b2a root_encryption_mode=aws - -[proxy:children] -proxy_a0 - -[web_a000] -10.212.11.103 hostname=web-a000-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-09e2b15106899013a root_encryption_mode=aws -[web_a001] -10.212.11.64 hostname=web-a001-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-04e63e19d14c9cce9 root_encryption_mode=aws -[web_a:children] -web_a000 -web_a001 - -[web_b000] -10.212.10.147 hostname=web-b000-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-085f4132f6ab1fcb7 root_encryption_mode=aws -[web_b001] -10.212.10.211 hostname=web-b001-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-04c20b4bab5c07be7 root_encryption_mode=aws -[web_b:children] -web_b000 -web_b001 - -[logproxy:children] -proxy - -[logproxy:vars] -swap_size=2G - -[hq_webworkers:children] -web_a - -[mobile_webworkers:children] -web_b - -[webworkers:children] -hq_webworkers -mobile_webworkers - -[webworkers:vars] -swap_size=2G - -[pgbouncer_a0] -10.212.41.150 hostname=pgbouncer-a0-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0582ef2a9c57fa780 root_encryption_mode=aws pgbouncer_processes=2 swap_size=2G - -[rds_pgmain0] -pgmain0-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com - -[rds_pgformplayer0] -pgformplayer0-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com - -[rds_pgucr0] -pgucr0-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com - -[rds_pgshard1] -pgshard1-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com - -[rds_pgshard2] -pgshard2-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com - -[rds_pgshard3] -pgshard3-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com - -[rds_pgshard4] -pgshard4-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com - -[rds_pgshard5] -pgshard5-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com - -[rds_pgsynclog0] -pgsynclog0-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com - -[rds_pgauditcare0] -pgauditcare0-bk-production.czikvdpwvdww.us-east-2.rds.amazonaws.com - -[pgformplayer_nlb] -pgformplayer-nlb-bk-production-17249f03f744a864.elb.us-east-2.amazonaws.com - -[pgmain_nlb] -pgmain-nlb-bk-production-75f4dcf6a91cd6e1.elb.us-east-2.amazonaws.com - -[pgucr_nlb] -pgucr-nlb-bk-production-c4336e567a87e30d.elb.us-east-2.amazonaws.com - -[pgsynclogs_nlb] -pgsynclogs-nlb-bk-production-af78decde238eb6b.elb.us-east-2.amazonaws.com - -[pgshard_nlb] -pgshard-nlb-bk-production-99be8739d5b85b57.elb.us-east-2.amazonaws.com - -[couch_alb] -internal-couch-alb-bk-production-62333703.us-east-2.elb.amazonaws.com - -[remote_postgresql:children] -rds_pgmain0 -rds_pgformplayer0 -rds_pgucr0 -rds_pgshard1 -rds_pgshard2 -rds_pgshard3 -rds_pgshard4 -rds_pgshard5 -rds_pgsynclog0 -rds_pgauditcare0 - -[postgresql:children] -pgbouncer_a0 -remote_postgresql - -[pgbouncer:children] -pgbouncer_a0 -pgformplayer_nlb -pgmain_nlb -pgucr_nlb -pgsynclogs_nlb -pgshard_nlb - -[ansible_skip:children] -remote_postgresql -pgformplayer_nlb -pgmain_nlb -pgucr_nlb -pgsynclogs_nlb -pgshard_nlb -couch_alb - -[rabbit_a0] -10.212.41.209 hostname=rabbit-a0-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-01640329a616749b0 root_encryption_mode=aws - -[rabbitmq:children] -rabbit_a0 - -[rabbitmq:vars] -swap_size=2G - -[kafka_a0] -10.212.41.30 hostname=kafka-a0-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-03e12c2f481fd8af2 kafka_broker_id=1 zookeeper_server_id=2 - -[zookeeper:children] -kafka_a0 - -[kafka:children] -kafka_a0 - -[kafka:vars] -swap_size=2G - -[celery_a000] -10.212.11.179 hostname=celery-a000-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0ed9962ffe8e16c86 root_encryption_mode=aws -[celery_a:children] -celery_a000 - -[celery:children] -celery_a - -[celery:vars] -swap_size=8G - -[pillow_a000] -10.212.11.180 hostname=pillow-a000-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0b23885a5a4daf320 root_encryption_mode=aws -[pillow_a:children] -pillow_a000 - -[pillowtop:children] -pillow_a - -[pillowtop:vars] -swap_size=2G - -[formplayer_a000] -10.212.11.163 hostname=formplayer-a000-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0adbcac28a4016ffc root_encryption_mode=aws -[formplayer_a:children] -formplayer_a000 - -[formplayer:children] -formplayer_a - -[formplayer:vars] -formplayer_efs_dns=fs-0287582bff16a9a0b.efs.us-east-2.amazonaws.com:/ -cchq_uid=3002 -cchq_gid=3002 -swap_size=2G - -[formplayer_debug:vars] -use_formplayer_debug_options=true - -[formplayer_debug:children] - -[formplayer_with_monit:vars] -use_monit_for_formplayer=true - -[formplayer_with_monit:children] - -[shared_efs_client_host:children] -celery -django_manage -proxy -pillowtop -webworkers - -[es_a2000] -10.212.41.16 hostname=es-a2000-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-04f32d6467cfc8994 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[es_a2001] -10.212.41.84 hostname=es-a2001-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-04a205c847e9bee36 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[es_a2002] -10.212.41.144 hostname=es-a2002-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-061bc80b47d4af996 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[es_a2003] -10.212.41.223 hostname=es-a2003-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0c62dcc83cbef4abf datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[es_a2:children] -es_a2000 -es_a2001 -es_a2002 -es_a2003 - -[es_b2000] -10.212.40.151 hostname=es-b2000-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0275feba1ef4a92fa datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[es_b2001] -10.212.40.91 hostname=es-b2001-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-07b8da671f6fd5bc5 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[es_b2002] -10.212.40.228 hostname=es-b2002-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-09b01fed0945164ec datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[es_b2003] -10.212.40.211 hostname=es-b2003-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0a7f6c9b5bd336322 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[es_b2:children] -es_b2000 -es_b2001 -es_b2002 -es_b2003 - -[es_a3000] -10.212.41.251 hostname=es-a3000-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-03963aebbab9f388f datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[es_a3001] -10.212.41.161 hostname=es-a3001-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-02d56fd603e9ae729 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[es_a3002] -10.212.41.102 hostname=es-a3002-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-079edad4c6f6e74ce datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[es_a3:children] -es_a3000 -es_a3001 -es_a3002 - -[es_b3000] -10.212.40.146 hostname=es-b3000-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-064a0657515defb73 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[es_b3001] -10.212.40.104 hostname=es-b3001-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-09a86e94b49b9f0a1 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[es_b3002] -10.212.40.172 hostname=es-b3002-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-04e547f605e4b447d datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[es_b3:children] -es_b3000 -es_b3001 -es_b3002 - -[esmaster_a1] -10.212.41.133 hostname=esmaster-a1-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-03450966b4641a16e root_encryption_mode=aws elasticsearch_node_name=esmaster_a1 - -[esmaster_b1] -10.212.40.55 hostname=esmaster-b1-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-04c5ebae4a1cd10ab root_encryption_mode=aws elasticsearch_node_name=esmaster_b1 - -[esmaster_c1] -10.212.41.228 hostname=esmaster-c1-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0160322ab9ae0b0b1 root_encryption_mode=aws elasticsearch_node_name=esmaster_c1 - -[es_master:children] -esmaster_a1 -esmaster_b1 -esmaster_c1 - -[es_master:vars] -elasticsearch_master=true -elasticsearch_data=false - -[es_data:children] -es_a2 -es_b2 -es_a3 -es_b3 - -[es_a2:vars] -elasticsearch_node_zone=us-east-2c - -[es_b2:vars] -elasticsearch_node_zone=us-east-2b - -[es_a3:vars] -elasticsearch_node_zone=us-east-2c - -[es_b3:vars] -elasticsearch_node_zone=us-east-2b - -[elasticsearch:children] -es_master -es_data - -[elasticsearch:vars] -swap_size=2G - [couch_a0] 10.212.41.153 hostname=couch-a0-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0113dcf847ca28abf datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws +[couch_a1] +10.212.41.62 hostname=couch-a1-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0dde79ea855218d9f datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws +[couch_a2] +10.212.41.29 hostname=couch-a2-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-06d41c8d2726ca675 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws [couchdb2:children] couch_a0 +couch_a1 +couch_a2 [couchdb2:vars] swap_size=8G @@ -312,13 +26,4 @@ couch_alb control_a0 [control:vars] -swap_size=2G - -[djangomanage_a0] -10.212.11.191 hostname=djangomanage-a0-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0a7d4c21b7ee0f4a2 root_encryption_mode=aws - -[django_manage:children] -djangomanage_a0 - -[django_manage:vars] -swap_size=2G +swap_size=2G \ No newline at end of file diff --git a/environments/backup-production/inventory.ini.j2 b/environments/backup-production/inventory.ini.j2 index d5f8b8a6fa..1c78bcbc53 100644 --- a/environments/backup-production/inventory.ini.j2 +++ b/environments/backup-production/inventory.ini.j2 @@ -1,213 +1,11 @@ -{{ __proxy_a0__ }} - -[proxy:children] -proxy_a0 - -{{ __web_a__ }} - -{{ __web_b__ }} - -[logproxy:children] -proxy - -[logproxy:vars] -swap_size=2G - -[hq_webworkers:children] -web_a - -[mobile_webworkers:children] -web_b - -[webworkers:children] -hq_webworkers -mobile_webworkers - -[webworkers:vars] -swap_size=2G - -{{ __pgbouncer_a0__ }} pgbouncer_processes=2 swap_size=2G - -{{ __rds_pgmain0__ }} - -{{ __rds_pgformplayer0__ }} - -{{ __rds_pgucr0__ }} - -{{ __rds_pgshard1__ }} - -{{ __rds_pgshard2__ }} - -{{ __rds_pgshard3__ }} - -{{ __rds_pgshard4__ }} - -{{ __rds_pgshard5__ }} - -{{ __rds_pgsynclog0__ }} - -{{ __rds_pgauditcare0__ }} - -{{ __pgformplayer_nlb__ }} - -{{ __pgmain_nlb__ }} - -{{ __pgucr_nlb__ }} - -{{ __pgsynclogs_nlb__ }} - -{{ __pgshard_nlb__ }} - -{{ __couch_alb__ }} - -[remote_postgresql:children] -rds_pgmain0 -rds_pgformplayer0 -rds_pgucr0 -rds_pgshard1 -rds_pgshard2 -rds_pgshard3 -rds_pgshard4 -rds_pgshard5 -rds_pgsynclog0 -rds_pgauditcare0 - -[postgresql:children] -pgbouncer_a0 -remote_postgresql - -[pgbouncer:children] -pgbouncer_a0 -pgformplayer_nlb -pgmain_nlb -pgucr_nlb -pgsynclogs_nlb -pgshard_nlb - -[ansible_skip:children] -remote_postgresql -pgformplayer_nlb -pgmain_nlb -pgucr_nlb -pgsynclogs_nlb -pgshard_nlb -couch_alb - -{{ __rabbit_a0__ }} - -[rabbitmq:children] -rabbit_a0 - -[rabbitmq:vars] -swap_size=2G - -{{ __kafka_a0__ }} kafka_broker_id=1 zookeeper_server_id=2 - -[zookeeper:children] -kafka_a0 - -[kafka:children] -kafka_a0 - -[kafka:vars] -swap_size=2G - -{{ __celery_a__ }} - -[celery:children] -celery_a - -[celery:vars] -swap_size=8G - -{{ __pillow_a__ }} - -[pillowtop:children] -pillow_a - -[pillowtop:vars] -swap_size=2G - -{{ __formplayer_a__ }} - -[formplayer:children] -formplayer_a - -[formplayer:vars] -formplayer_efs_dns={{ aws_resources['formplayer-efs'] }}:/ -cchq_uid=3002 -cchq_gid=3002 -swap_size=2G - -[formplayer_debug:vars] -use_formplayer_debug_options=true - -[formplayer_debug:children] - -[formplayer_with_monit:vars] -use_monit_for_formplayer=true - -[formplayer_with_monit:children] - -[shared_efs_client_host:children] -celery -django_manage -proxy -pillowtop -webworkers - -{{ __es_a2__ }} - -{{ __es_b2__ }} - -{{ __es_a3__ }} - -{{ __es_b3__ }} - -{{ __esmaster_a1__ }} elasticsearch_node_name=esmaster_a1 - -{{ __esmaster_b1__ }} elasticsearch_node_name=esmaster_b1 - -{{ __esmaster_c1__ }} elasticsearch_node_name=esmaster_c1 - -[es_master:children] -esmaster_a1 -esmaster_b1 -esmaster_c1 - -[es_master:vars] -elasticsearch_master=true -elasticsearch_data=false - -[es_data:children] -es_a2 -es_b2 -es_a3 -es_b3 - -[es_a2:vars] -elasticsearch_node_zone=us-east-2c - -[es_b2:vars] -elasticsearch_node_zone=us-east-2b - -[es_a3:vars] -elasticsearch_node_zone=us-east-2c - -[es_b3:vars] -elasticsearch_node_zone=us-east-2b - -[elasticsearch:children] -es_master -es_data - -[elasticsearch:vars] -swap_size=2G - {{ __couch_a0__ }} +{{ __couch_a1__ }} +{{ __couch_a2__ }} [couchdb2:children] couch_a0 +couch_a1 +couch_a2 [couchdb2:vars] swap_size=8G @@ -224,12 +22,4 @@ couch_alb control_a0 [control:vars] -swap_size=2G - -{{ __djangomanage_a0__ }} - -[django_manage:children] -djangomanage_a0 - -[django_manage:vars] -swap_size=2G +swap_size=2G \ No newline at end of file diff --git a/environments/staging/inventory.ini b/environments/staging/inventory.ini index cee0183559..e69de29bb2 100644 --- a/environments/staging/inventory.ini +++ b/environments/staging/inventory.ini @@ -1,222 +0,0 @@ -# This file currently contains a lot of commented-out hosts -# that refer to presumed eventual hosts that will live on AWS but don't yet exist -# As they get added, we'll uncomment them and the IPs will get populated from live AWS resources -# using the aws-fill-inventory command. - -[proxy6] -10.201.11.133 hostname=proxy6-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-03426413ed3c925f1 root_encryption_mode=aws -[proxy8] -10.201.10.72 hostname=proxy8-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-053e8c3dd100eb276 root_encryption_mode=aws - -[proxy:children] -# Amazon EC2 -proxy6 -proxy8 - -[logproxy:children] -proxy - -[web13] -10.201.10.244 hostname=web13-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0fb86252d8c3490e9 root_encryption_mode=aws swap_size=1G -[web14] -10.201.11.231 hostname=web14-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-07e90679fb2045342 root_encryption_mode=aws swap_size=1G - -[webworkers:children] -# Amazon EC2 -web13 -web14 - -[pgproxy6] -10.201.41.151 hostname=pgproxy6-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-073f7e3a3e2eceb72 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws pgbouncer_processes=2 swap_size=2G - -[pgbouncer2] -10.201.40.250 hostname=pgbouncer2-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0fa48da862fff6dfb root_encryption_mode=aws - -[rds_pg0] -pg0-staging.czkracjslrn2.us-east-1.rds.amazonaws.com - -[rds_pgformplayer0] -pgformplayer0-staging.czkracjslrn2.us-east-1.rds.amazonaws.com - -[remote_postgresql:children] -rds_pg0 -rds_pgformplayer0 - -[pgformplayer_nlb] -pgformplayer-nlb-staging-98b68467f6f5717b.elb.us-east-1.amazonaws.com -[pgmain_nlb] -pgmain-nlb-staging-fe77ab2d143eedbb.elb.us-east-1.amazonaws.com -[pgucr_nlb] -pgucr-nlb-staging-b32586fcc9999b2d.elb.us-east-1.amazonaws.com -[pgsynclogs_nlb] -pgsynclogs-nlb-staging-1102365451f68242.elb.us-east-1.amazonaws.com -[pgshard_nlb] -pgshard-nlb-staging-e8ef6b4c55f3618d.elb.us-east-1.amazonaws.com -[couch_alb] -internal-couch-alb-staging-546646078.us-east-1.elb.amazonaws.com - -[postgresql:children] -pgproxy6 -remote_postgresql -pgformplayer_nlb -pgmain_nlb -pgucr_nlb -pgsynclogs_nlb -pgshard_nlb - -[pgbouncer:children] -pgbouncer2 - -[couch24] -10.201.42.45 hostname=couch24-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0caec26b9cfd45ee5 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[couch25] -10.201.42.181 hostname=couch25-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0afc91048d3e95f44 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[couch27] -10.201.41.18 hostname=couch27-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0611558a44278d575 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws - -[couchdb2:children] -couch24 -couch25 -couch27 - -[couchdb2_proxy:children] -couch_alb - -[couchdb2_alb:children] -couch_alb - -[rabbit_a3] -10.201.10.231 hostname=rabbit-a3-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0054851ab37af66d6 root_encryption_mode=aws -[rabbit_b2] -10.201.11.253 hostname=rabbit-b2-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0ab4cfe0c9ddc9c70 root_encryption_mode=aws - -[rabbitmq:children] -# Amazon EC2 -rabbit_a3 -rabbit_b2 - -[kafka15] -10.201.42.91 hostname=kafka15-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-02b1111b949791b87 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws kafka_broker_id=15 zookeeper_server_id=15 -[kafka16] -10.201.40.77 hostname=kafka16-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0065c2ad36219b227 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws kafka_broker_id=16 zookeeper_server_id=16 -[kafka17] -10.201.40.229 hostname=kafka17-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-006ac04f7f57193d5 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws kafka_broker_id=17 zookeeper_server_id=17 - -[zookeeper:children] -# Amazon EC2 -kafka15 -kafka16 -kafka17 - -[kafka:children] -# Amazon EC2 -kafka15 -kafka16 -kafka17 - -[celery_a1] -10.201.10.31 hostname=celery-a1-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0e15f23fcc71fd3c9 root_encryption_mode=aws -[celery_a2] -10.201.10.163 hostname=celery-a2-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0fcf2866598d1a9a6 root_encryption_mode=aws - -[celery:children] -celery_a1 -celery_a2 - -[celery:vars] -swap_size=2G - -[pillow5] -10.201.10.194 hostname=pillow5-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-024e31574a4e10811 root_encryption_mode=aws - -[pillowtop:children] -# Amazon EC2 -pillow5 - -[formplayer9] -10.201.11.71 hostname=formplayer9-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-039f848c98b6bac65 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws -[formplayer11] -10.201.10.202 hostname=formplayer11-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-04ea09168ac760aab datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws - -[formplayer:children] -# Amazon EC2 -formplayer9 -formplayer11 - -[formplayer:vars] -formplayer_efs_dns=fs-ba70cd39.efs.us-east-1.amazonaws.com:/ -formplayer_fsx_dns=fs-0ffde07db112793e4.fsx.us-east-1.amazonaws.com:/fsx -cchq_uid=2002 -cchq_gid=3002 - -[es18] -10.201.41.168 hostname=es18-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0904ccfba6eaed16f datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws elasticsearch_node_name=es18-staging elasticsearch_node_zone=us-east-1b - -[es19] -10.201.40.228 hostname=es19-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0efd33ef8e82292c4 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws elasticsearch_node_name=es19-staging elasticsearch_node_zone=us-east-1a - -[es20] -10.201.41.254 hostname=es20-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0acf18d9eafc42268 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws elasticsearch_node_name=es20-staging elasticsearch_node_zone=us-east-1b - -[es21] -10.201.40.13 hostname=es21-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0c5137e706cc30e8b datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws elasticsearch_node_name=es21-staging elasticsearch_node_zone=us-east-1a - -[es22] -10.201.41.68 hostname=es22-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-08eed79af34c48103 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws elasticsearch_node_name=es22-staging elasticsearch_node_zone=us-east-1b - -[es23] -10.201.42.104 hostname=es23-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-02a5ef51bc2a6cdb3 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws elasticsearch_node_name=es23-staging elasticsearch_node_zone=us-east-1c - -[elasticsearch:children] -# Amazon EC2 -es_master -es_data -es_coordinator - -[es_master:children] -es21 -es22 -es23 - -[es_master:vars] -elasticsearch_master=true -elasticsearch_data=true - -[es_data:children] -es19 -es20 - -[es_coordinator:children] -es18 - -[es_coordinator:vars] -elasticsearch_master=false -elasticsearch_data=false - -[shared_efs_client_host:children] -celery -django_manage -proxy -pillowtop -webworkers - -[control3] -10.201.10.21 hostname=control3-staging ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0764b662e2e2637c3 root_encryption_mode=aws - -[control:children] -control3 - -[ansible_skip:children] -rds_pg0 -rds_pgformplayer0 -pgformplayer_nlb -pgmain_nlb -pgucr_nlb -pgsynclogs_nlb -pgshard_nlb -couch_alb - - -[django_manage:children] -# Amazon EC2 -web13 From 7a0cd69ce8469aa2ecc77d6f0732478eca9b2f77 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Tue, 26 Aug 2025 13:03:33 +0530 Subject: [PATCH 05/25] have a basic postgres.yml so that terrafom plan runs fine --- environments/backup-production/postgresql.yml | 126 +++--------------- 1 file changed, 17 insertions(+), 109 deletions(-) diff --git a/environments/backup-production/postgresql.yml b/environments/backup-production/postgresql.yml index 991a4624a1..8eb8fbaef2 100644 --- a/environments/backup-production/postgresql.yml +++ b/environments/backup-production/postgresql.yml @@ -1,119 +1,27 @@ -DEFAULT_POSTGRESQL_HOST: rds_pgsynclog0 - -postgres_override: - allow_dump_from_pgstandby: yes - postgresql_max_connections: 1000 - postgresql_version: '14' - postgresql_checkpoint_completion_target: '0.7' - postgresql_wal_buffers: 16MB - postgresql_max_wal_size: 2GB +DEFAULT_POSTGRESQL_HOST: "control_a0" +SEPARATE_FORM_PROCESSING_DBS: False pgbouncer_override: - pgbouncer_default_pool: 490 - pgbouncer_max_connections: 10000 + pgbouncer_default_pool: 290 + pgbouncer_max_connections: 400 pgbouncer_pool_mode: transaction pgbouncer_pool_timeout: 1 pgbouncer_reserve_pool: 5 -pg_repack: - pgbouncer_a0: - - database: commcarehq_synclogs - host: "{{ DEFAULT_POSTGRESQL_HOST }}" - username: "{{ postgres_users.root.username }}" - password: "{{ postgres_users.root.password }}" - port: 5432 - skip_superuser_check: True - -# We're temporarily disabling read replicas -# and will revisit after migration to RDS -#LOAD_BALANCED_APPS: -# auth: -# - [default, 1] -# - [pgmainstandby0, 3] -# locations: -# - [default, 1] -# - [pgmainstandby0, 3] -# userreports: -# - [default, 1] -# - [pgmainstandby0, 3] +postgres_override: + postgresql_version: '14' + postgresql_max_connections: 300 + postgresql_wal_keep_segments: 0 + postgresql_work_mem: 1MB + postgresql_hba_entries: + - contype: host + databases: replication + users: hqrepl + netmask: 0.0.0.0/0 + - contype: host + netmask: 185.12.7.167/32 dbs: - main: - host: rds_pgmain0 - pgbouncer_endpoint: pgmain_nlb - pgbouncer_hosts: - - pgbouncer_a0 - - formplayer: - host: rds_pgformplayer0 - pgbouncer_endpoint: pgformplayer_nlb - pgbouncer_hosts: - - pgbouncer_a0 - ucr: - host: rds_pgucr0 - pgbouncer_endpoint: pgucr_nlb - pgbouncer_hosts: - - pgbouncer_a0 query_stats: True - - synclogs: - host: rds_pgsynclog0 - pgbouncer_endpoint: pgsynclogs_nlb - pgbouncer_hosts: - - pgbouncer_a0 - - auditcare: - host: rds_pgauditcare0 - pgbouncer_endpoint: pgmain_nlb - pgbouncer_hosts: - - pgbouncer_a0 - - repeaters: - host: rds_pgmain0 - pgbouncer_endpoint: pgmain_nlb - pgbouncer_hosts: - - pgbouncer_a0 - - form_processing: - proxy: - host: pgbouncer_a0 - pgbouncer_endpoint: pgbouncer_a0 - pgbouncer_hosts: - - pgbouncer_a0 - - partitions: - p1: - shards: [0, 204] - host: rds_pgshard1 - pgbouncer_endpoint: pgbouncer_a0 - pgbouncer_hosts: - - pgbouncer_a0 - - p2: - shards: [205, 409] - host: rds_pgshard2 - pgbouncer_endpoint: pgbouncer_a0 - pgbouncer_hosts: - - pgbouncer_a0 - - p3: - shards: [410, 614] - host: rds_pgshard3 - pgbouncer_endpoint: pgbouncer_a0 - pgbouncer_hosts: - - pgbouncer_a0 - - p4: - shards: [615, 819] - host: rds_pgshard4 - pgbouncer_endpoint: pgbouncer_a0 - pgbouncer_hosts: - - pgbouncer_a0 - - p5: - shards: [820, 1023] - host: rds_pgshard5 - pgbouncer_endpoint: pgbouncer_a0 - pgbouncer_hosts: - - pgbouncer_a0 + form_processing: null \ No newline at end of file From c18fc853da4670143091a009bda9d9653dda1045 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Tue, 26 Aug 2025 13:41:02 +0530 Subject: [PATCH 06/25] add back alb which was removed earlier in inventory --- environments/backup-production/inventory.ini | 3 +++ environments/backup-production/inventory.ini.j2 | 2 ++ 2 files changed, 5 insertions(+) diff --git a/environments/backup-production/inventory.ini b/environments/backup-production/inventory.ini index 1dd01749bf..a9698c6d89 100644 --- a/environments/backup-production/inventory.ini +++ b/environments/backup-production/inventory.ini @@ -13,6 +13,9 @@ couch_a2 [couchdb2:vars] swap_size=8G +[couch_alb] +internal-couch-alb-bk-production-62333703.us-east-2.elb.amazonaws.com + [couchdb2_proxy:children] couch_alb diff --git a/environments/backup-production/inventory.ini.j2 b/environments/backup-production/inventory.ini.j2 index 1c78bcbc53..4717418bf5 100644 --- a/environments/backup-production/inventory.ini.j2 +++ b/environments/backup-production/inventory.ini.j2 @@ -10,6 +10,8 @@ couch_a2 [couchdb2:vars] swap_size=8G +{{ __couch_alb__ }} + [couchdb2_proxy:children] couch_alb From dc118c94466d039c96c898d8f394b79aa1a00134 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Tue, 26 Aug 2025 13:47:03 +0530 Subject: [PATCH 07/25] update known hosts file --- environments/backup-production/known_hosts | 93 ++-------------------- 1 file changed, 6 insertions(+), 87 deletions(-) diff --git a/environments/backup-production/known_hosts b/environments/backup-production/known_hosts index 3c4cb702d6..a3fabbe328 100644 --- a/environments/backup-production/known_hosts +++ b/environments/backup-production/known_hosts @@ -1,93 +1,12 @@ -10.212.10.147 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNnVBJB22LPB1wW2FKHoJlQMCXNdldqo7uuQHHTb9soj/oZYxG6ksoqmLdGQ9zAca+5cVD54J3vLvV7kFN9ewzc= -10.212.10.147 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJPVNzS6vroy6cCIoKeTRFknlbe4jORJLWFCkvcceO3J -10.212.10.147 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0a7UNQKtQ5FVcH5Uvs9wf2r4/TnbrDYYHdU6TMKbI/syp0VoXG3G2RKbikrGO0XTdxlhbWYgW7CsTCglLeWo3S3Wl44rH2QBL03PnurEJ1NNdrrEhOJRc+nypnYLkg9+jKnxpgc9kvhV72AyZg+NWEp/C3eeZIeVJyJJ/F8/2bmVHSymvICi1zAqZfF/RleoaCsbpjUDFseQCAZ7a0vmkHEVh1XDKNwaFtojVSv/IKrOlJCyEKlzqngM7Hv049HTAu9tCQvBgOZ6wex+BmXcqLhzjaoD0/2XBiQex9zjB73Uk2K+LEduRJo1TMgm1m7oVvrvRkW4o+jlHXeJJ4y4ga8lEciDedbSUyml9GB9LMyQXZZLlT9z7uVVUt4TVtxb056wh6PMTkVCtkp0+Wwpd6YTmfUd7CEWtq59Wyal9KgoWF9p1LkkTYyHW9zvycu6194JMKQ0NPQfPdW8qc2IfjwFmQoZJIlxExv0u9VQIDdmWQGvdtaDmqiQgi1kcPXU= -10.212.10.211 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEuAafKv5iCdguAngjE5Jjj4k9bZFB0DY6OJ2uR0qhc/s87KlURojapYJR/2kQwIFPqlByy2wkvVJbhx31T8QI4= -10.212.10.211 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIbPKMQRiKWYuqWCXoeXgGcFCYnzQoGYmmC6LzKAKqey -10.212.10.211 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCdQZMpe0aulNwy9wPQTBp4M/5JaFmdV/Q/S++Wd3723TQictyAxT5kY1e8T1iPX4mxOtimVr5GqSM+MPJdvBUQ0JHtJgbOAVE8XJXypwyeOlLt+4pEh+s8SuhFyAfNvRKk1lSSULr1R1helPnJmVm8m/5JPHvE1PA+YfdAV8+EviSU6sSPHs14DzQPnjbCHlplaN09dCq9SL+GS3909APcIpsu4GmPNvYIyK3zfn0UwPSpRwYobQf6+KPyLxb/T4Sfa8mq6hoXsltHrIEN3Ccs3Th1HWByukA0vd1Ch2ZoFpPTVdZ821oFrMiK8YuXI1JcOuJu+nWW9l+qPwR0+HvjW/1r4hBKF7FvBcGOgXlohVMf6QM4FgNafdWws7EpzVgfnLDlby60wK6HTfqCUwEWZBumuAFXdnIoRjnxHSR2xlV/94IRr8SapczCR9GbzhvxJtN1vnXrVAgl0+ku0NCJFEj/IAFyG4sZHEsluCHmNvz4oDRpkENjrhmIn5t007k= -10.212.11.103 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEGs2Rr5SrugIR30T+hf6BNnkd9/eOI6Q+xsRQHrzr7lXTdDtmC84N2DJd2+yyR92EjdFbjDeRInBABlK45k67Q= -10.212.11.103 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKP74Ub6H7pnq04BHPL3KgKC+W6/0KQ0yv+Zc1pTQJHW -10.212.11.103 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDVagmqE4YS6opink7KyV7o4MaObcneJ7iD0Se2EU9PTiHnu/keuL0zqTs9deWcblXyaZMAW6IOitUWJKDPMDB/C1DZyCxvUr6lNpgbF3uoZgzIoGaMvEQUHi+w1kw5dnUoJet4y10Sa3V1zKDAqn4Gyb90huohkgYQVWJ8lA/VpLL2hSksOeY5XH6gxm59ovOmS43WEs0TFT8JyTZrBhPGpeFoa369gzmxjEOvU5bFFsHFV6QdpYW8VVMHGFxxmyegZMiGJ95mZScheAkC19TLKn1AInHV/4zO3dwn1s4c419auOiRy9drwXoAoNW/QlUIMC0uL/19whoeLzNd5lia4BcAXLT0br5dTOzy/KHOx2JVgnLLfLdLlkYjw49kcYzMZVfY8ar8B6/RCYihRg7/R+V/tcAxu+LbrWyAkizuSYlXAd79iZShk6v46K4xPSMespLnKc28wC/b2zqQbStqtNX7Y/fIqNftzWJ2Q7UdDsMtycgFOcU9qALnkl0Lin0= -10.212.11.163 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCX7TuwGYgtMRAh7Do0b/VB1vFYitedj/a30DnEcYkKcmI2kH5j31RsknGZvqdao3OA7THbePIZtWW77Ia6silY= -10.212.11.163 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKH/uEWXxvDoRxWTdbdBKc/uuAUHMbMh2sDJ5iFjezdy -10.212.11.163 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCg1u0S6W1naVduPJvqUMqKGzIA9Bu0FSGCCNMaiw6MvApjMYpPsGvqK2OcmjnEKzuo9X0zAIyFmzPVHjm+vJ/WiKy4yo4gnaNM5UFXiCYU3I5ZmnYE2GpJR9LIp8QqeuvR+WSqlZPpNJTT21dlhHf59DL8lqwvj4ERXBCP56zjKbN2fTDfpsdUgMEq5h9Yjq7QiLLKrr9Ytl7Cbf5FmMVxlftX3yH+0KzxV8Nmb5uncVcMY1urLI9gcfw1Hk+kEhKNTvyPDgsGzEtysFk2ZvEL5zH+SoRppPuFA+KIu7pbLQYsLOArXOIKbfUSC+lZzNEQhmVLvCEztan/m0gzFmyIySCgQeAdAcecLVfL6DiJHx2v7/sSSmSfF0TEOYPK2YedsSMhsxMvOiaba/2ASSqE8vxB1WBL1L/b0HmwuDgK+0wH3Cqb7213HNu2rmjGAGKeXeU+gUQYkhqHEfn+g5LwA1RbqLqFPCnv4toUXrY8Kk1f5Itvk84qnVjdsdgGnOc= -10.212.11.179 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAcCxD2QqVv25LtLMRJ1GxtJrA3fR175xBwvRpV7yMJ1HScgrTU7dbLlJ9AAQUyh+rdcdfo+x3mTEqxK4lYiAHw= -10.212.11.179 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIDLMoTgqJ5q7dVvkAAFOyQRhT27Uhts682pEyHMRnc3 -10.212.11.179 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDe9DeVgaBsPlgKOGG+NueTIPrdViQVN88cIpO/R9Q96anshaZiyCGK8moo4kQih5QtlKikPMSaSmmPYcTfei9EzOq7GxjtK/4J38LFx5Oqt5cyFfB1BL+W79fHAKnHqwY4X6Sno15MIbhUlejHHLouonSJkwKIUNfmtZHBKjyz4Tt8ch8qewBZG1Sz59VQmrGu8z/TMyE6UKbrfmZQU1yINI3pMROUBlWcGHbcQHzbFZe3C40mhsxm5S8Z1HV1Gw6pjYh/q4b6FYexybOY/UlyiT9R/snpPFmq3Ju8SqDTFP0IYpfuI4FEramRlxA1H4HTYrutBWdC1s+FFhDW+nE+J9RQhfyfkYBXvR6O8V7I2sOSNQWsRxRtswLuNjwzfAjtJ7oVrwA6Wr4nu6n48pFBdEDIj+QS6YqeqcyMu8d2NA0kOx5ufe5tdzmoksKc+ywTK4TrrMbh5BJu0dngWKfgRYAgOI7vEu5Uj+grbKn9QMDA9rj5jePst8Oqge2HBUU= -10.212.11.180 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM5OcczujRH0hGtDzVatLEcvLRTZbX8s9mPy9banJ42dR5XYyDJfrhk5nvy7ebAdCkHdcAvGCR/pcJawjTuTFWc= -10.212.11.180 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHi9XqJQC5uWzCme5EzsNw1GI5/xivd/t/bW3kYRUjo6 -10.212.11.180 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAJhSpgte30KMI6yM8D2Uyh8FRpYjWgVCAFkXc5FvJCN3dEIlxeDaCcL8dpI5GxVhq7PPSTB8rb+YnstcMQqa6eAx+FneoENPr7QzJGKpXWAod+QAMfQandxdq5srRjoThVbdOmLeI+PFk/c0V/QK/2qtFyoiE1wmlJ7gu5BeABLIyEP4QVY/wFsCFFr5hErgItPx+gp+gCgZqFYHRmD2Bs6RFDxO8wy0Tq1REQsZNyVQnTn3mJ9PQmtkRW0ChfKq7c30H933adqZi699cyY6noHjyKhpx2Kjmi6Zb4GMS2hQzA6rTeY5fgOO/BOQbm7G50MW44UR3Lq4oNOIT23M3kG8Xcd2Jsi4oIMIAARGMe97Imfa65QEcLm7/GogT0PTlp6zuX54YZuudtQkAL73260hY90jVTCfAUoLlUS4q7GLJwJvmMhrYnUtEBih56mCrGk6PMehXzqNtjarCIMhRyvXRCEH2QmKfd+TL2Tx/3t4in6491wUlBTtkvUeGBpE= -10.212.11.191 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFXXA3TEgRxPF9neBdCyzKwGDOiod/Pll8M9GWBOoiYcnQd90qOigoU2VYrvJDja+0ct0qU+IUWdhQqA66+YXRo= -10.212.11.191 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB5GTjjNVkYovlsPz/y22WSzC0Dm/P6qrLib9WyBUevw -10.212.11.191 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCljF6zTG8ChpwCgW+q7dR9UEcswu4yJ2viha2dd7B6hXP3kd9ix4dfW4sOi1WnQ/oc0/HWfmsCCcseddIg7sJysJm+o2xtIrBPqqfquP/QJNHChW8zQSwPD6M42o6SqyC7ivC9wi7iIJ++z533W2EKo+dSc4kGOTEwMXeSoidMatK7DW8lnvOTtWctH1RDc6cMH8RbN3aAK48ldYTMSZYWfPWoI4XFMrhczf0ihonyQdhRK8lceDrMDlFK/ABfPwjrpsJfVsgX/W4x8pOExacxA5+uo9ad7vTkR8Qb1Xz/Yt6242iiULzgFNrVg8RqLHDDrhUE9qJEZUvftqu3+LShGDODTiBDKpUGeGMxoDFoIrrXoLO9pipTUBXKkUHFgI2rBhELV9EM1JNXT5J59S6ma9HFz9pnSpvkTL7QC1DKQfiamf0m52CkByBhV3usxbS38VbgdQhGqA4d32RCCcg7ns/9+j2GQpo6qsZSfkDLccy2euDLnoeS8NUKAx5G6yk= -10.212.11.64 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBID1bD+piBEOGDk60TiX7tM/R8SziCsbjIVbkPAavNRpjKriOE+VBYBotJIFGuiLgJGRfY1y58NaTZjqMeiYFks= -10.212.11.64 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBy3/CyzGxiwFioxHHCxfh0zd8eXcITV1eKQMRKTlrT1 -10.212.11.64 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCd1qQAhYPwX3I1KoQbj1aEonr0BbW239749yDRz6v1stYzwIv42UXOvDVI8C3hbV43yVPrELz1MyfDBGSpMwKvObF9nsgsv/OJzFubFsn/uyBqnZ88GeFWsGy5qGJi9zcqeK6IOu30qABOSe4BYWErEffdkPzblMh8WhyYouu3ZtOEd8cpQSQT1vzKpg7zlmAw0HssmmAZCreqMQtmh/imhEFouicV4dsWuhIvuQ238iYhEn0PEIr6rzGTglx/GHDq1j3Ug8Km3GwrCndX/3RW6PrpbYe52vYIKR8eTr/Q+K8ZyfdurnLUIJqJQPz6gQOnm/qZfj3BuERasIFYiWheCtR0GucKET/3uo0M9P3MQ2UXMd+/5jsYrS2anZi2wUWM8HS9yJoaz1w1W7XPL0X2Ses/q38cXO67F+p6pFv0deFyDiZywh5/4nD08DCQwtm9vQlAzyd1sbbkYSl8XEiqFj8MxXvXf6UTF36E+pwi/NuuQjCXuZ419oQ1Dirfjlk= 10.212.11.9 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBL8UvZUKCFRoUQkgsxBRR+tFYgpHP2m+Aj1cXB0B/zWmJRyVHqG3nW1Ag9C5M1dnClPk9LyClRnkZ3RTQXNI2RM= 10.212.11.9 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILBOPX2IitCfC8ZvzDuV3iDb89IzFqm2/eMQjHwSmSsJ 10.212.11.9 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDQBS6O4yHFXVhu+2t7iffcKDCu2h5WDYZt712YuLBmi+W3TmRrVfXM6FzoB7+REMB9MwRI2sM6Vt8kfbLvf5timrH1EP7lZY4oWvZuaLg1JtPAwIMk4icOLbQ23flgCD8zfKVLnqnmhqg/uXsJo/JKJFJOyWyGWnEbP4VbEDa/4ur6FT8kUZUGczF1ITpf4bKRXrd6MuIes4vyjy4789TrGIetpV3ipJ/QVvxHO7mdCh/AocznWnzgGBjSAsX9Hv24Jm1dX4zHowt6dALa2HQ9mjWoRJAAGXZHxCQzszh6ZWo9sw+0SoIlzOB3A5Gq72EK5MhJLhZzWRgZGkPOWf4gCri39Z11ZtK05W64YYHviDUyGaS+B8RAJSdE+A8dIFCn2qAor8Y0CPJ4t4M8nq7Av7obiCOdcdrePl2dyOXFA/aq4Ste8RtYfS9iu8QCUEfwViteTiLAX79V28Ox+waYYiq3/XHJd19pvaoFeYSgAN/pyZvZhb/VBDt+H0o0NZs= -10.212.21.205 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKYf8/5BhtXny+C2A6pL2TPZsABYj+7Sr1Ughc2GYaVSaXks6JVosGvuUtM5+pBZX66Pzsdzh5E15FH5CbHsdJI= -10.212.21.205 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMMJhAnYpyLRx71Cvr+684zg4V3DNuM80fnxizVWugnu -10.212.21.205 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCqj9+km+PIRYQXk+VRUBzHz1jNn9TpLKSfHUvuwCw9by2Dk8Flmdy5rOTxjyqpeyBi8wGwyhG5AZsCxfZ73TKC7zzfvwQfEFndnOmnnNwBOCGlUy6W65DVOMLmjnGzo+jiq+IqBh91Wy7z05gkVYu7EQTG1GeBwOL0Rv4SvNVQdBK03yeAXuFRuBHu9JKzUkXFzFl3Fs/WsJJyIrlzKat/75Fv8kNi1sZ/Cx+CEvHLcfX0YE+jnkcx3hf6hE3c1V+8wTQEutYaLZf/rTysNfvY8uxxroLWjXlnMSsP3ukQJxNT2rk7nIhKIbQPXrFkD3jQl8B03Fq+wTC1lfcb/qA0slAs/M0Fqa4YK0MMhbdklnOyEJZEhOxhaK1cMcX2OrDkLdpO0K06AKsqaKkXQCYknkFmdAXceAmjICuvGvMeDBHsHcRcURI4D9oAfvWg4oAfw3B4umarvUEA9S1FtAn1qkx0VszyH1I3Nlwl6M1G6hojJg0Tcevoo4QbkjydNlk= -10.212.40.104 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAwBxNeaEVcED8ZMA4IjxwA1glaJuaa894ULeNyuMt1GCWNMb42tqIK7bAcaYGa0xLrzaBWiB9gJeUMlipilnlk= -10.212.40.104 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP4NTkJs1bhdr7IMo5NfV17XSgdmghePVnZCh8EfYpCn -10.212.40.104 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCVRn0ZRoT94vorCtckFrZ9aB8AC352wJp5rs6xxOcchqRT+w9tJuILkQMhPObihOYfBEqXkl34n5S8j7zlURoZ1nwaHYJCmnp9XF3wwH/ITFtg1+i7syynTYuxAmcU0VOP0DyOBm63ZImWR/+CsysjVkLGNhVBTZpMpYtc10+9fY4VeXmDTBaNgLkei1N5bFLhH3z7WWCrETdQvbuWisr3BdcZCE+YACKs5yWoN8JFPdG3h7azImaMpMdpYmUfDIOEClosJbBOxENaZXkkLfrHZl+Zh+KIa+vayp9h/mv7QZXP68jxlBkHs0+4rorIVN6TBykFJjAqvb0vpbDBmAjnRfUapmbeFhHIGcVFPjfM9xIJnSGdjejLY7BCRJnCZPT2LtWquV2v6jNYrWFMX1j3E2kpaVOCbj6svw63N/tSpV1oskghFF+Y3qn3YRD/O5Ielbss0WJDBc3gdMgbKOgqYAv4k0ls48SjSW7RJ7ycZkmDQefpMxtIYdbeOt5WtFM= -10.212.40.146 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNGFZYT1htjtXb8/naJNIRr36oefb6FaY5FEE8o2v0ajxuHfet8gZL/k/fkdAGehjnxYT07sbEVFI8QBb+8Me9Y= -10.212.40.146 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICwJvFVctvPn6wuXKtBiDMhs0/djzc650GgOJghs1//T -10.212.40.146 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCxOH4fKisWYmpkD1cbSVXrRmS9alEFcLMjhdYHnYmZ4cBfsYJ+HBkZ+ZmP5Nbq4YWQdPwMnZzxIVEDpaWCSpQGAb7J5brQLdbK9W06WOmFoXx2gCMuhLRNPuns3srlEwo4ySPjV2PND6mI1+blS2pKcOcYEUaY6gmu5eENVL5Xap9KD2Ip/2N5xWzusXpGjY/QoypNpeAomuOIiDodeviitC6+x66NPEjabVVYnawqh7MEORGPvWDNIsjM85Q0rktrW+5dQ33a0eHV0GxjeHAQ0hk2nCkdviheuHktEtwVKpKgeCfh29KyX1zKXt5orKZZFxXfMscSuBUpJjmpzwARcneX/7L8MimCW4pUfrbndLh9K0t7FpSWp/TucWHQ/dWAEiKTAUhYEadn54/orIpv1pxQ0/VzNXOuqR8wGIWl0qpb7tFioUx+q4vMjPBHzoVVF4imiAxCiutyaYZcV2i5G1rxmLv4PYICLw+NzV6mWaaW/2Bq2K4wwYEQoZznMgk= -10.212.40.151 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIy6K96jJYwwKxivJsUD0mR5uoDHTsYFbLUdCLINj/jnMrziN5OTc4zEM4vwz5WCjETrrT+qYsiQnYHFHVVwg84= -10.212.40.151 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC/I9+Xn/ZYps6SyzfQspLZslL+7SiBJDX+BmgC9kxpp -10.212.40.151 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtUU2Gi4VA6kEQa0i/gj5cO2dUSDg5/MyuF+MicxEzqLJ5O/GymRJeoqHsbMjuDix0oHVOCGh0kbNfHomQ8hZQr4kab2Kefqd+EprnTjfRgUP1ijS2CXVQ/fOjhAeKPSQLBeTeqC7zPbC5FMWH38fzCQyM2BvDuw3fQjxFFRsv6vQxcqUJwVP26/O7D0HD+zdPhAN0MQhmdZgvzz3jGIFtYnpppmECnNho+fGTkYApyxJ8QrJiKfRqpsauI3vjTYbZg3PYVYsMPn1WK4q42XcLruErXq4b8ILMbVevEMbnoXuQ6fjNh1OPbYBq/bTq/bVmxa2VAbxzX1rxziUdrmb3kceV1Vw8YU65ibXyen0fAKPXoPgjva/FqXIn4T92n3JgXryrDeDyrI/P0QJWRBfuMKuTdBAxUk0V2CW1zBHd9x1/+iEe7E4lVUKhavyW+86lQGHHJ8tmqVy1MNSgS/HHyzSklDbxw3L0MXd7HfzfzsxtirZoyPMmXwlCMOEV/Vc= -10.212.40.172 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBTXl/V7BUDaCGd30CSRRExIz61JRhNihe6DvIvC8f5u8A6lCKdiaJ4AsvZDGGiAbp6rIwtEdkRJ/JrTqgDCpEg= -10.212.40.172 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILPnLEEfWqq8CP4xN69pRnhMZgz4k6mpenAEcn2+jhEP -10.212.40.172 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvlfZEQ8rilVGefS3VCEFntwVdIl6qNfru4H+IW0PBoJ5LVJ8Wq62adLjEe4+6aK3sMXHYOqclzUYYoMGUJaXBYa2LIfg/Lj53aKXr3EvMUZDoqjFE+TQJD9pS02rOG9Jb9wGHvxcruDY+imYGuC4xdI+4kC0AEG/CBCXOcXJbbKJOv9uU4zYACgDemzQQD4H/E9lSxUrJpmZOI/GoPtP80QrbMnJlqjzRVrpeQh053PthkCksFV/L0CuX3lQ9roRLpSC2oRCasHTm2puXk8EOj0bXKEErr78GGQYM0/+vsCFHx/GzMD+OZR0o/WbkxYq33wEkrR2sWrfPrBCF53LIJ668SaR0h4nkGqhzPqpA1n/KvpoZbEaBDpnAxHPONS+nTq4SShvZ1PjwNjf8MjEDRySCOAhxromphsyNdTwH4LZIcgqNQIWVKOAD+Y86f1toQn0+bfhfcRks6e9BJBCmpRifLOFTWqJAKLqjzicmpmYcj9/pSJqCE4kJnIAz/X0= -10.212.40.211 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBUm0mQ82VkVYxiVfYhjfsp1fC/r5CW0UHiT98MtXjrpe0ew/nKnnzjaferyYmMAwbSGom1VvFQjS1aStdeAHlI= -10.212.40.211 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+tnSuhCphqqARZ3PMbCwVF7sItQ7gQ+LzzapgNf9gp -10.212.40.211 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDmSOxaLWLwcdLMczmNwjZ23OfEUBqDFj3Srz06lcBInkpmUIOFuzKnitaKR29mRpmCa18XGnzlDK+YD5Ik/OwAIO1AYLS51CjKF7W7j1XQsInqpZjlVMMeWmST8NTJ41QqDIKtSxWgrN1sfMOa+DJUXWwdbcZO9cc3yiL1Ro1p0V8RSEoEYxxHEDtukBuRkj2QFmqIS6k4ODq/xM1xI0XvQZZeRbV2Qt0T7vynG6Be79yfSFvQd75B7vSTD5hCD7N2vGeF8I92wdDdyNTpQPbdpTgdE9TlRWIZx0L8E7xNTnyud6ibjkhAb8M7pqhwaTVkte8ML6ZEGNo2jM53TuZkZTey22VCoNzQPAQHqXDBo4JbMdh0SukaXrpBZlgSkUWssT5cIZWqJGKKWDNPDIEkpvcF5mEzh+ZrYVx2EFPiGUa35ta7m9Io+76Owa9BBWiPAjO9cV5NQHayLvUgcyenAXPrepVezqSfEaohrsE1WLDU6XPn3pr8l372QrKkxhc= -10.212.40.228 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBF6Z/rY1GE8jpl2PulKD/rojfqzv5sW6UCcWSBjLmWW8iZNEduNuwJP3EGa7rtnMjpKseyRb34CAta6TXWf9CvU= -10.212.40.228 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKE3loVjAtAoMyRAkb+Pnq5Ozwn0uO/u8TPyh9Br1p3r -10.212.40.228 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDkgRf3v1M6AFUAMTBqlXDskrwXNmBBMmN/6JPL1nRJ0FF6+8rkaG0MWF7dE/MqZa0Dj7qdTP5CN8dCampaO40wG5DWFTXVvF1lmTkfJuGwg6hyq9KieBYZZBKzHRu43bzgRFkafY/wFD2rWEt91fxKxi8KjBa71bITFu/SqbrTQrMjdjFPP5d4fd5og2gleBsM0EgG7kGzUE8nkTqE9LlhTUzS8bejWyiCWqssVw8CQaG1uFxMp5OOQuSP1psf6TS5tAdiLTHCHnyRtejA9aiKy2hm5iOKbcQgyKQWRh8UarLOiVH8n7uRZDrdbORey7r0sz0HUeI0TJhnarG7YwT43IEtUeF/I5HOcE8t24PA6AVoV8pnEZeA8Zlwpqg8tKgEu8n6X1fWX25CTtaxzD5NLWtB7cxMaymtTp5lW4oM93PQDhO6Nwy9PN2T/nU+wBJd5CUEzFUamZCrbWBNa1Ozk2B16DjY+2ZwTnj0jswPAfH2kdOmRCPUJcmPh+pgUiM= -10.212.40.55 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBB32KbK5zgn7hLlrKovsQgfZ1MCi52lb6StqMXW7SXEiNlMqcZG6keBdwdlDg5pExhdFVHgkwgbje6tydv2MINI= -10.212.40.55 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAVfDr+SbkcMVgjw9xGmR9tenIEInOI8COaD7V4Cj8H2 -10.212.40.55 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCyNxA4w65z3k5ntcgkLv5i9BPjuSpYct8/Q4+RRnzFC0KtokPgpYFTvbKYPSF2Tta81F9a7hpfGcEhvCwqiJQRDeXGqUWrfYoDfCw9pfv0VKJxh3I2Yl3gRlN0mYcy10y+VPVtZsPBkSwFGyxAH1nCTnz1EAgFL6/SRYxbuMsFGrdNWcRlJFGGPzugzhHAKEv/Zcu9IC1tECGhU8s4kMvQSv30UafSowrIYyt2V6aRKd1dmIVL8WDMTqao8t60Wyg5eH0UHYie2yDbWgRXTWCThKgh1SHms+LsVmpu3yHqBrAS+9xA7dWb0pyG0QQ5B72a2Pp5M9qPURxClwKtk1xT5kJ0riTQboUWpnMRfsfyWGx48Ov5f/VqJwQtgl2krlaS+CNoMcz+FFAOYThtlEK516Zv7BbCqBaDYUbBIRmh6pc6p8pMiqNoGfOrOkja97c3gfz1dFKYb6v1XNK1NZ6GeFsEhQ61L+jzVdl4xONXLpkrRCVL3O9CLQwZ740DHNs= -10.212.40.91 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBL/wglI1agUHWv0I5xXWnTufGVMUAuu6pUTifSVTelIl46p75k0RahwxHx1l3Pz7uVqOSwLuy843D0cF15mqb2I= -10.212.40.91 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILihszbcrId6NDAM/Ddd3ekhMJotUhNB0mm0UKjA2j01 -10.212.40.91 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCmMJ3m5cpJgS1x+Z+wxXtmHggV3d1Xo+30j5c8prXMD0IviGBfDdE70xmk3EDRNISlltrowhOGDyYr511tLLL/3b2LxTYyxw9S7Y569u6T3toFpJDXnmdeLznuAGfS8D8Hx4asODpxfItm13hyEJynH10ebI0MUb2ky2bhN7dBrhOxo8nn41psSjNGzRKcEmQkL2CkPsCnx1l9o+owrnR7VmcyHHHNlVQaVXtJxUrAlvoWDQIZ43+uOtF6ucdVIrDcqpRwVjuQ0rXC/pLIfTFhbDsPN5jp3ixrU6TX38OED8DpH4DrW01OWEQloBwcBt9UTZFEEvgj2uNIJY8Y4rC9bC+mtDHYZAiLhsjgcM97ig8FSxqG+uc4x3VNtmlA6fe7YWkB7BAHDRdFO5jLjkm99TgvB3u83OSfurRRjWwCI58kamDq7xNFXUT6lgQAz2aCXP94tlaLdx/JkROb9/kkUGKAESTv98Q7eJN9LAy544cH1k4sjq5v8krzh0ophKk= -10.212.41.102 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFd7CzefJA6T1z6sIr+Ccvyk//3BEi550c8PER3C5zGpPwJhxljxw7J2nTlT3BFPavDZZ3AE0sTyKx70U+D+6x8= -10.212.41.102 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA3whTModVfBTNzKdm3lPMmesXNDLGlnrry+y73WX2hu -10.212.41.102 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCiBAcfZfROGTs1zZv4sQBUoOOtOw5FowFDRk3v+HOuIST0HfN0EmcJa5qlIpE4pqZ8eooWBXz13+Zw9lo4kHGqMoNn7fksw5tDR+fSZHfXDIDJBdgqkcN803jUXMRJJHoIaYWY6O+A3HRZeiyDwXfpzX+PlkLlxYn5TW1QGKKNKXmKFM0rjna7Ybqfjm2KWb5Er+0s7yS8YVh2RcDf67KODHLF8Z+hUzUMpKvNyY6sM0+xwHimNZ5J3U6t0k+frjc2CNDKSCyKIV1yBBEzKCa7//+iUUaKOdi5PNSQzkuAYtlSf18guBn7nAba/ltegY/k4kCP4Wm6T7EcK1YCsaoq7FGm11tou4OGK/8BX6ag5mDTOFyy3kIxwU71Ssb10xh2BkOnxy9VfFFUTlE19/wmngiBo6YVtGNsF0wPvaTrgjWtBl5Qqk1R4KGnmmhhubaI4WaiZRIWzgiUlAka7p2Gl47/fxrJTwxrWYsUF36eSrcv21SyLqLQ0m2B/V2wBV0= -10.212.41.133 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOQnakg7o5CsFiHzbezYoeJRdm3bTTwVSH7NkJspeEwXaS2rWJvp00qkkMW5eb+d9UFueMLuenuKeBmeDkLjUDM= -10.212.41.133 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINsvQ7qqHi825FdzRGpWdCb11tybzV2cDyhrCnKVqXdw -10.212.41.133 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDRK6B1SXyEwaon95fRDcXA9YVtiJp+AqxbSIETv5TGNT3o4LxHVdvB3GyV2qSnZhiSqcnEY0RCBf+AoABKZ/yDZFvK93/6lDZa4c6KXK1qBKmuTLU2m+OQt6zcL55Q7a+fUi7TzVO9kV0BHdFkChZOOMRq2teDYRm1ZF90P7pME2EYo3fFNyKeCL+i9v6Y3z50N6uZ/GML0sjR2EI/890iWwDNQ/+EnsZ9Ye5P6Q71LclTPfUt2oaOvrBjAHAOE/YrcEmqWGCGaC8X2f89YJb2XB/p3yzLwfJ28DzwDhQph37XbHXgYzwrKkPgvDcNX7Bc/81Xesm8/k4rziFMsmIWXSg/NjZrgFE0k1jC5VTVfAcorhlzHaOymY+8plZywisNX5DizVsQzuf7DlK/MCe9h762HWcWO8uKZXIo7Qfc4AL+f6E7NeK327wwNZnPztMNRGIuEpWRBdNzH327/YUx2p+OB1D1wbpBr8I8wKmLTDvGynNPRYE8etaPPcNnXhU= -10.212.41.144 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEJ90mNRxRdqmU59hhw9GjnJvqBZueSCgo1b89oG2Hzi9b0ZAt4uooA+WTIyemRrUH8GyTpq6mo9dG/tbEyn6N4= -10.212.41.144 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMtyyTmYk7N/VX99/lAtvIzDLZ1YCB8a3MM9/DoVSTE4 -10.212.41.144 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDn7yB5JLKs+lTwxDL3O9xJGyyW/HKwlDiFSYWXCifDzlcTNfmBjKnK7WM72kTA0PbaeAHvfYvhS4KfXtargmKCX7TVr2/wID3kySPFRK2xbhVTp97mkrGlR8o6h2sDW7eyoSCs4YnlUnmzFmaN0YOngOSao6McyzbQUHOJI1/IIUwULx+lhZywYDTuR0EwvtBJ1nGeVK0SLCHU2Ovt2jciKIZeMx1EpoVaO6LWnQEcvmgxt9odm1Y6mCnwZjFFfrIEWZohOqiGPdWzZ/2keaUk/r7S83sWFkNKlFwKKhSwFJoBWa4roEypC8AXgnwo784XnrJultAvMp+SDbWOGk0hFgD5yyy2weDGEE2TCyTXeKWFtkPiSajwnp14JbIxJns5LE2icOEkTo5uVOXkIMuQWyctgUMe/f/HFId9hw6eE17Gy/3XvNOIyMlybCXMkR7AiPWQnOTAyXoO6mlkeFPL5bY2pmoi3AIGKXmkdi8NOnEgAureNtaebJdWBel1qVE= -10.212.41.150 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPieN7YDd3YVJ0gNLBmw866DAOnb1M2s5oH2BOmOmj5iHjoA0/pqK3AT8RxFtZAmf2czp+Tzx6lQZcAad+GB37U= -10.212.41.150 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaUa7Z4tDRWILFTaqiPbSZVjp+bLTanK0mGKgnWxLdb -10.212.41.150 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCeU6trMlLSliqX5l2TUQyBY9mdgkoJeXiFdbbdNNV2S6EofVqpSNToAB9YYxggbpmAGS4egbmlN2UcSOg98Vzu/T+oedDzcF7HhSVqJR/Ty6X0jtG60+M7XFe4krUNZGNO+RCe9nkLbF6MeN3PdUUIWNXdMW20JIB+qdygrUq81qzEz/uwLQBiN/ib5DwqyMf+Z+90YHyZESNtxdg/kcwnRkg5zAm3hwoCmOwJXJWQtgefMR3gnN7OH7qS2aTQ2pNiFmSpW8K+jrriJoQt0JcUALrZ6305mcyAmTWU68jKO65UJzjF2zesybftm9xQbRAMciexa7UGI7+BG92CWK0/YuAZNHY14gKzbTORIiTxPK0fNz46+oUScnJ/3JQ2jg2nFtacei8bI4OFvfX1HNE2TeAmNAl3CPDk6C6dRkASUYpoTVh+OxqZLIahKesXwq0uQJB06poNCb0fO4zCfZta0WzVBo08PkfbDL215SbpjOdwJo12u8RgLGsI7J6NQUs= 10.212.41.153 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBDCwyP1nSSs5iTbIwgRo+FhvW6HhXcrxuMOb/umz1q/WrC5ZXS+NOqrl90wv1paECjVhYBJr0DE2cqQEHV8wrs= 10.212.41.153 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA5aawps4jrRvaeP1WFYqFpvpyFHzTl8R8qM9iYi4Nh9 10.212.41.153 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCzffJ4c9rp4DLXwA3GfT5rJ5pPzQwCCLDr4nxDwdI476NA1VvOKo83b9IgCYHpIxritva5vgeKQM1AUxdUHSSEMSCQ+USySJLGm9LYgyuDuaw20p0nS4DIQL5AbhmhKgxeqblcEJVviLuakGzNlvn2dmLM7BWJjjtBh2/eQSIr2EXQH41Ik6SpF8SS9E0YM9BRjtfSZ9oppAtCmyyQ3aCxFOPne5S6XTPRrsncextKRBPQ0IJGR0EPVR32VLgaZB3BRxefNDTGFQ9mE9w3gFe3ojLkoBd0DbDaD96FwTAZHJUbtjEHDXoLU/IGgoAkCgQMPyybcGlmigFnVid/5pgleYWcf5vNS929YkyNvkZO09l0ykVy+3VqVnOvOepFxf/UTJFE5CzwT8oKYCzXuDhkexyTT0G2+yswe3r9gdN+JN5BMCzudo6Vpxd67jfaMaoA01HhSB6PYlxNHpgq6dpcZJTM1eZYEnzvF43OwH+z5BM7DK99pf6J2hjssW5GdPs= -10.212.41.16 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGFHZO3hJw23NkstIR4nzRNOEP+4i23engFvP8nOoXFsc6GQFH0xBwET79KgLx5tb593Cz5hvro50EM5g5Skgxo= -10.212.41.16 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJd2oVOmSKGXVX4Y6QrBKDrjI3yxDSObxGU4m+em9pZm -10.212.41.16 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDHqB8W+6ZxM3TCPi8kSi/bMAEcJ15wuNzhc12fLmbcfCmoDeWP+b/KLSt6CAm7as/72k9okwi9dkCo/bqk/p5J1ZTvsjQXk0ZMoAMT0SQJqQqgbjnIE2vD/WHfpDNExTKfrcNI8qVnIHITozTGukmFs3ShoQXEwFP9QbllYydGjaRO4YdIPeD0Jq6On5p8Mcn5yoSvq93b2BwCUC2VaA53+JS49H7Y4k6t36z8pADiQSh8ZZfN2GdN6svMHhUA2XfcC+6ZikUqP5r21ZD45qxGu+ZA9XydSElCVPoX+Lqanm/ylLQBhPrZx8HSKOiGeShJYVVrWuEzI4B/oEsILrBpmwzD0HFd6+lOB3HDVeEKuWZmjEw+ORStehjiSgBDlDFsly4AiZ67yaHLeWoesQIaGctlj02xXLEcisMnlKEiBh2Rye42gEPgb0i5EbGVyRCa7oZdl//UbrR9PbLCwMPgw7cq5NE3H9L3mtLJgCajFL6zr2SSgcqDw5UDoXPtpk= -10.212.41.161 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNsESsYVtp5LWlxOi1v/E8JB97rQCNg2pigJXYf2KhNlWNY2L5fl6sxWPL1oNI1p4OYiCCtaNHCTyYbpKRpK84s= -10.212.41.161 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEjyeKF/x+K9UiPau+8MdjjcmwkqZbRvpfVw+gkxtmCO -10.212.41.161 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6Ce1dZtgQ1tlqeFKTmpy5/tUuKtOsTScRW5N6VAtgGBfIEsKhZd5V4+D0nDL5PjfQWjNQ1osI0Lyjw+6Rgi5FwexcT8eOsZCRKdQ/ipWEqpXRUwXx7F0KJvawMy3H7VN9jjbjhvM8qgALowNarFJuDuNlLKjh2pRGWrsaZVzS7f8KWFuumkmueD0LwVS4Va4Te48Lxdx2cN2lRNDFflMRSzwoqbJqODlvxdlv0KBelDKuunta6R2XgYBADbMuh11OlY9pjXj27SK/co8nZzQUV8o5tX693ub4LbUv7r+L+S0PhClor+daLr27H31BGoawwEwGRNM5DEADiFKNS9jUVEC1zXtSYhMPEZiFFbOdTc7KncUIvxq192HbxM7K82pYQql67lQ42uNQxrqk3qGrGmZPsPbuIlwqfxFz0QMNrTKuwFalCgdo93JlFTdwCRhblJOS07baQm/d1KY4rtcTX7h1y6KcSso8QhckHFE/6pSFgHthCev57voh2Cdlxo8= -10.212.41.209 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBF7eO5xJkFADS62T/GxcHAE2BZr13azaGWKSxDG6DOYzk9AwB+2DwVvDvQcC1L/RWBlxxPPWZHU2npf7pX6iXk4= -10.212.41.209 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFxvpNlUFlrW4prlp9UGZ2vIQ9VYjHBw8vlVjR1TVg/2 -10.212.41.209 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCrFjOvpHAx30ifp7YoFhj4ckBdS6VVc2xu53hvRVXQvV6uvZJCrSWbQziaXGx9z4RxlER9kXhlGdSXGLtrmuesVSVVI0MwqkRda+o48AFayKFuMoPrIW6cfZg/xanAt8zyVXw+ERMCwY9eb9dlFAaFdqT2/Y0aAmqTdfOZ9rU5kGOIu7jVQkcVcupYsYmwkZBmwByZBD6sqM1eHdZbrFBXtqWkZTMZUK0Eb/gOHD6yGJDJ5Kx4S5g5pxmv0zjkzmz5pb4y5diObOW3yyFA9Tk3Vy8Y5/W+ntoGLD73DS8tFia0DYifsvQdOV1v2uTePdM2l8O/O4crJGuDs5hmzCBFWnhmzbDh8pmHuTr8mqoce1uGzZf0cesKdRTSN46ZZw9SkGx9+gNnzHBRx5zZX5+PTBcwpwMDlcCPfjSGhPhOi5emQBQSJloTslE6iyl9L7mFY7wo5Q8XKE4Qfxuo2NMtp/JoGQ9mDOzmdKLWx/oDJjixzg6nYiG6aQ2F+qyucEs= -10.212.41.223 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHpSfWY2qPZebEAGRQVLdyoGxpEELLwm4xdOHDYWerc5lv7wlpyraZf9vvLdxO4b6+wtdvCjkgYJU95AEH5eCQg= -10.212.41.223 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKUOckdqc11wZ/b7bPNhxSZvVvCn79EayuIyWWapQIfn -10.212.41.223 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNE1KvC+wyaHoRCNgsb0CmFYuI18XnIr1r+BEA/UoKT2c8YkscdG4EMZloO8300KbLczqOx5IsivXd0ra4XqqhJqFaXPH1aGubLNaxpfiG+EY1qx/IaSSNkhVGKHY+/FA2SlOEayhm4V2B2TI8YiC6VmXuOihu25DmUVicupQMnmzXUX1a3PQQUujU1bTte8OzOfic2nGqBBXAiCtDLgfwesSaJ1bfbx7jGuxuzNzzoN+ApqmzaIXt3AATFuS5p+9oU2u1U4/+zMyicLeOQzRrYY2Ah/eQO8jloNFLLRIwQHw/sRmwm2WPbNJgFMtw4W+cGHsSwoqDsxSMisd7DPGqnQWN4EAkAHWkQE1j4fAzZIrWHQKu/HQLODEtZ6ns17Opi/Y1yj9l1Zyx68JDrg39ZTRkhuf+jRSjiV1D9bh7TPVQx4msdjpFVczeJJOcIiXKcuFz6iPZE04bIre+Nw7GWXJVPlYAvr/7hFmg+z080U9e+EXfl2avVlEDd+J+Epk= -10.212.41.228 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHyOJT0378s8BdBRM6K0w9grsaVs3ejPpKDS5Zl823JkxFWmKBKotJZk7Ni2O5FeO1RNAABnggx4HrxyLoIi2xk= -10.212.41.228 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPy9yvo0Mjia1vG5wF07OKhbXNY01dnp9Q6g317fdWv/ -10.212.41.228 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDS6rQ2LM/EogtKIu101CXT8Rq7r2rF98zD6Kazisx1CtUWXRyZntlMQiSClxuRpkzoLiGvgmfRBoT0pjNuR3ZQuXX2bb2Pa1ckuhxBdPiOmDtTt7AwWPr8M5TsAGAIg6OLZZl2U4nTEsVvJNYTZMVSB+VprrrD2NtNIWm2YLxFGqiUIryZi3v05YtHGz5FAI2GLEy2IoxclyRtX2Z2WAmyTK0riU7Fbe19xxxbGMy6MBxhVNjGmaZCZz8eEZ35sB0iKWPw/3PqJw+WjYv9zv7vQy/4Smau+UVPh4fvei6AXSYVfiVotD71GKM75L+N5cKUaLU5JWkhPAvvnPYrxZaxcSxTOREMwGmglOxVtZ9aPfaQXejrzdFgWVNBcQOdzP1FaEM+7kRi0UULp/ZXwXGZ51DYzJAm0Y6QQjGWwPzF8XPPcYu3E88khE0Mq044OXp21SVtcFP9ruXtEmXCsnWF4UR4hPPlp4IQyuj/1+HxUivnDP98ckpzdw6znJmb03c= -10.212.41.251 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP7bUjeSWHEb1oA1V3MV9d03+dlOYbepf7VWAfbm/di408llWVVU7fDG/IUnPa7ePY/xZwtPGOSlw5dLldcU8uM= -10.212.41.251 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJE+VOU6NZJBC8Ke9VOcijD4Bw9HUions/cvQ5Wh6k+/ -10.212.41.251 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDTJ6nB5Ww0v1o09tIcPbQL6lmP4JcZgJ9bcLxqxwfrJtIKN3t7bS/xEjTowxR10hmZamiTBTC6kndBsHEocjzjZPLma8lkhDeoyMemwWgPg+RtotCJ4Ly+RdC6SnU9xqAuXip83PfebrQaMx2cZvXcJJ31Whirh+96lkPgnKiEWsrmRbpi37JW6HCUPO7OAXtL5zGCoZFRExy27XiqwqCXKUcpb5iTVEQvHo7rJHQPJUtsdLwB4KN6PczBOe7NKwazOhyw2SZEUupfkTJqeIKb5YSYdSxXmV2Xdm+mk4d9LfxVCoINBU4euRNGx6Y5TJHdJGiVzXx5mRVhQY8EJUvbpJNuMdQvnilNqbJTzvfC0SGKNHSQFk8L0HR9/skqP2KjC0j6fOoKZgphImIMRGps2NpHVDs/df/3XUPt0y9kAsIqcBsd1QYvAnP/3xKzFkFU1+bELsgtTXdDut4whHFf//BGqXupjZT3PA054tJc44D7D6TI6wiIVuSPoqMMBo8= -10.212.41.30 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDzXheA7qyESBf9kdttCxCh5m3jdjk5+aajnl2s0C/SsZ7uVxWu4kwrZNMdORirHUbTCZlE9CkJQSjwhHxHh9+A= -10.212.41.30 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL5EoIJ4m/7rrYgRsAHO5MuCn7JNJ52Uj5Z2UkUIAqxo -10.212.41.30 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCscoKwImXtI+NqN5sBfC1fa0cdHWheePQu55sPqRJTGd0bJ594GMBCf8Njv7VGvk9qbzoolbTX5MRBeUuLhO8Zo8BO4WNmICejen5EvCONRkCtAbkJFPgu3goVyCQINIMhEafbg8sUAyB152n3EYRFw+vnz4ci2wT44xpCnQniHeT4I119ifaEeXHRCNyaacUPSw8HRU10PCIO8y5Q7E5EnkBr+2pvJEaw0xTDJueMtFvJq6JWBblHaIHL6YV7d7vAPv+pwYw2X1BGX9d8NWsDiVptPI3sWaWKb3XRBfG72d+Zp9c11vVRm4ccXRY9iWw2y2rGU4/lxKQSKTa/MPJ37tWAF27uYMCA2wgR4dyRdcekb14vmnWhXum5Jdvc4Hf2cPZoe46JeBk3+d/WGUma6R1Cp7wiZTrWQZ6MfKSXB+MfZvVWuscvIBRihp91kJc+8nX8Dy5LTYtzLZgMJS8DQWyXqesde6O4B1uslFWGnbYtZX8W2kK7gCyPVZhyTNM= -10.212.41.84 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLYQFV0kWl+2J7S0ZBu5exAddq8u7IIWcIq/ZIDyHGnV3FkCGGfXtiRXEiNmfb7saOKQbMNJYQN1i3ySr5PXbWI= -10.212.41.84 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMlsnJfC5rg0z2XFYWxv2PJkVnHggvxabjyUCDtADf0Z -10.212.41.84 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDiDpdExNUiEKhGquVhr12SOgxjPKxfqha0lvNSfSmO+wqBIRYo6MapW/0kFZj+fmclb9xtYuTfVDyYEpF3TYLaNqrcFpGty8qvhuUHp0RXIozIuHaMCTmDMH7uUtIScYIswizGngUKc83AbgU7vzIG5flz7cK6PLimr7ryYqqgAVj/kyBcQ37GMb5hvoyNIWFIdjZDespjirjw00er/Z+3QjLDovnUi0P8J8x7ufw+RHxQrzhIVd006TJtDDFm7oFtgg8i9AmtZIMF/o84MUjBvQSIvR5xgeANNUnpaMxTAjhP44way/dazTZj7Wc7VwW19/i4O2shaKnn6sQlyiKKBxC/TDQezoFPx00R8cDMbyRt6S2hXYaT4AgDcAlVFKJHCbfxywjgL3eubck8erEBW0nm24jQFEGL+/e5v5kDxDigvMd4LJJPlw5v7ThTk9WfSdLpmLrkrRYDagJZvxkyKcea3W85bruAfGXkIFw+pLOfoByJflq9Q6Dtnmnr1gs= \ No newline at end of file +10.212.41.29 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBC2isCcPR33ML5bIYLXHMlIlkqJhipZ3ifDKwwbhPN1qNJD/mCSqjdhKbVK3Y40+ppCpp7J+6E3ycC142Yy+zyY= +10.212.41.29 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHM/4i0u5ouEkoGJqJRe6mNhx0Rlnr0OSFkCUYLEC5Un +10.212.41.29 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuVUS0GFVKj7Z/z3OUHV2W1+Spih8TIqQI2PHePt8pjcW+6A0Yki12aS461vySw63Wf1JJk4oaibHoKZ0rGN3qVWMw2/IJf1MO+ToVlamLWIVAR0cGfACJfsRKTdUVtRSnwuFUDkJ5la+54bxNse8XkPe51Pt0BMOQfpZEUpVpKMGmlEyR+jg0ej7chf6OK0w+u6n/pDB/VAbchcY7Q5XKnUSfKwZF+mLVRXqo4AWT9DAZHmEy3VbeSMOYJp62kopDamM3YRYpww9Zn3LoVgpFt5hHUAIl+10mGh1tvyjU2insmBEla2n4lHLW4+8CdkaypjPaEO4DwE7WS0nBrbDhODo7jaOCF1mZpZmO7UTuciLpCf6OdlIkqm3USW8MP3Wiav0wAMujJMEBpjPlX94ql+peXAU9AImPg3xQS92uYrP5MDRiVuIbzm8N5du1fNlyKwBBPu+/z/ERNPF89Q8vNLTFP7sZtwEl4yH2X0/LAG3pamV9HeMPOg7YC5A6FRE= +10.212.41.62 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFtVHkVi5ng4ViAxxnELn9L49Ws+xX3wJGZiwnmrfunGVRMJWxKUm/e132gHFmRuSX6ZnpmV0VTOGjHI9QvSk5k= +10.212.41.62 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFjUFthQ36Mm+E6Ztw31fA9YGKRlHJEFmOfVJ/Eja/Me +10.212.41.62 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCsGe1eSjvQiSxtczhaoCfTkmh0/yQNx6/z6ISBao1xkCA32ti7oAdZxsDRb9Ehm9/Z/HBWdUoTJg05fSLdxXLPCyP7YjksMT8kdk2ODimYTyVKTcoFAkrVbIYiKEMAh6a8Un6HwOc6Kl4bUuohUAxVJhE0rEHlhH1nAng0eiLJhJn3rKxG2j2MwAu+Kp9dv5/eQAtEnIUhW1iFU2uOA+eFjxXze/v25Iia1qOxNuaOLnUsDYyNDDn1JXZdSM2iyvC+Bg1/Ffb2GOJ5iiPOUe4P/ZHmZwJBuTgl0b7cd8m3ic/K9ZPM6ZbewkfowS+B5ZV2YAPw3zT3dS62xQtrFM7YpdISdYFTVleChwdv1ErzX/bumDSc/KyFF/ZCCR2XvNQ+Lv6qNFKiqJEUpCM6hi+SsAudbl00As5hH02jJ9icjgHBsa2PnJsnfOAZoIhJBNSuS1quleltFfMGjmcY/R4K/oqKQkgRQSwz7iyz/r9Otd7s0vvUJ9NxcVvg+uVcBS0= \ No newline at end of file From de42d5222e2717a555d4b03e2f727eaf45fd6d70 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Tue, 26 Aug 2025 15:43:20 +0530 Subject: [PATCH 08/25] use control for every machine --- environments/backup-production/app-processes.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/environments/backup-production/app-processes.yml b/environments/backup-production/app-processes.yml index dbbf6878bc..0855fa217b 100644 --- a/environments/backup-production/app-processes.yml +++ b/environments/backup-production/app-processes.yml @@ -6,10 +6,10 @@ gunicorn_workers_factor: 4 formplayer_memory: "1g" formplayer_g1heapregionsize: "2m" management_commands: - celery_a000: - pillow_a000: + control_a0: + control_a0: celery_processes: - celery_a000: + control_a0: background_queue: concurrency: 1 email_queue,repeat_record_queue,analytics_queue,reminder_case_update_queue: @@ -23,7 +23,7 @@ celery_processes: beat: {} pillows: - pillow_a000: + control_a0: case-pillow: num_processes: 1 dedicated_migration_process: True From 37b87c242eef7c22bedd183c5b02ab23591b31e8 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Tue, 26 Aug 2025 15:43:50 +0530 Subject: [PATCH 09/25] use control for every group while runnign deploy-stack it was throwing error otherwise --- environments/backup-production/inventory.ini | 38 ++++++++++++++++++- .../backup-production/inventory.ini.j2 | 38 ++++++++++++++++++- 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/environments/backup-production/inventory.ini b/environments/backup-production/inventory.ini index a9698c6d89..49ccfa94c8 100644 --- a/environments/backup-production/inventory.ini +++ b/environments/backup-production/inventory.ini @@ -29,4 +29,40 @@ couch_alb control_a0 [control:vars] -swap_size=2G \ No newline at end of file +swap_size=2G + +[proxy:children] +control_a0 + +[ansible_skip:children] +couch_alb + +[rabbitmq:children] +control_a0 + +[zookeeper:children] +control_a0 + +[kafka:children] +control_a0 + +[elasticsearch:children] +control_a0 + +[elasticsearch:vars] +number_of_replicas=0 + +[celery:children] +control_a0 + +[celery:vars] +celery_command_prefix='celery -A commcare.celery_tasks.tasks' + +[django_manage:children] +control_a0 + +[pillowtop:children] +control_a0 + +[formplayer:children] +control_a0 \ No newline at end of file diff --git a/environments/backup-production/inventory.ini.j2 b/environments/backup-production/inventory.ini.j2 index 4717418bf5..d9dc922b4a 100644 --- a/environments/backup-production/inventory.ini.j2 +++ b/environments/backup-production/inventory.ini.j2 @@ -24,4 +24,40 @@ couch_alb control_a0 [control:vars] -swap_size=2G \ No newline at end of file +swap_size=2G + +[proxy:children] +control_a0 + +[ansible_skip:children] +couch_alb + +[rabbitmq:children] +control_a0 + +[zookeeper:children] +control_a0 + +[kafka:children] +control_a0 + +[elasticsearch:children] +control_a0 + +[elasticsearch:vars] +number_of_replicas=0 + +[celery:children] +control_a0 + +[celery:vars] +celery_command_prefix='celery -A commcare.celery_tasks.tasks' + +[django_manage:children] +control_a0 + +[pillowtop:children] +control_a0 + +[formplayer:children] +control_a0 \ No newline at end of file From f4778a8b7dc6db33084b5a18541d5665b90f47b1 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Tue, 26 Aug 2025 15:52:44 +0530 Subject: [PATCH 10/25] add leftover webworkers group --- environments/backup-production/inventory.ini | 3 +++ environments/backup-production/inventory.ini.j2 | 3 +++ 2 files changed, 6 insertions(+) diff --git a/environments/backup-production/inventory.ini b/environments/backup-production/inventory.ini index 49ccfa94c8..e20cead792 100644 --- a/environments/backup-production/inventory.ini +++ b/environments/backup-production/inventory.ini @@ -65,4 +65,7 @@ control_a0 control_a0 [formplayer:children] +control_a0 + +[webworkers:children] control_a0 \ No newline at end of file diff --git a/environments/backup-production/inventory.ini.j2 b/environments/backup-production/inventory.ini.j2 index d9dc922b4a..854064f8da 100644 --- a/environments/backup-production/inventory.ini.j2 +++ b/environments/backup-production/inventory.ini.j2 @@ -60,4 +60,7 @@ control_a0 control_a0 [formplayer:children] +control_a0 + +[webworkers:children] control_a0 \ No newline at end of file From 44fd35be78c48c66e141e599c47eccecf81ce6c1 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Sat, 30 Aug 2025 19:44:13 +0530 Subject: [PATCH 11/25] upsize instances to increase speed of replication --- environments/backup-production/terraform.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/environments/backup-production/terraform.yml b/environments/backup-production/terraform.yml index 39ea6d47e7..fc012b187b 100644 --- a/environments/backup-production/terraform.yml +++ b/environments/backup-production/terraform.yml @@ -26,7 +26,7 @@ servers: os: jammy - server_name: "couch_a0-bk-production" - server_instance_type: t3a.small + server_instance_type: t3a.large network_tier: "db-private" az: "c" volume_size: 20 @@ -38,7 +38,7 @@ servers: os: jammy - server_name: "couch_a1-bk-production" - server_instance_type: t3a.small + server_instance_type: t3a.large network_tier: "db-private" az: "c" volume_size: 20 @@ -50,7 +50,7 @@ servers: os: jammy - server_name: "couch_a2-bk-production" - server_instance_type: t3a.small + server_instance_type: t3a.large network_tier: "db-private" az: "c" volume_size: 20 From 3f693b05b2d49c6f79a82d48011641d93c53fd83 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Sun, 21 Sep 2025 14:10:34 +0530 Subject: [PATCH 12/25] make couchdb_cluster_settings are in sync with prod --- environments/backup-production/public.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/environments/backup-production/public.yml b/environments/backup-production/public.yml index 0dcec31947..3d61954b70 100644 --- a/environments/backup-production/public.yml +++ b/environments/backup-production/public.yml @@ -97,9 +97,9 @@ couchdb_os_process_limit: 1000 couchdb_reduce_limit: False couchdb_cluster_settings: q: 8 - r: 1 - w: 1 - n: 1 + r: 2 + w: 2 + n: 3 kafka_version: 3.2.3 kafka_scala_version: 2.13 From 5a378df6af04e22f4a0a57a0421c77b5c60d8dfe Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Mon, 22 Sep 2025 20:06:40 +0530 Subject: [PATCH 13/25] Add kernel settings for CouchDB distribution ports --- .../ansible/roles/couchdb2/files/vm.args.defaults | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/commcare_cloud/ansible/roles/couchdb2/files/vm.args.defaults b/src/commcare_cloud/ansible/roles/couchdb2/files/vm.args.defaults index 4c8089e15c..3fadf7acc1 100644 --- a/src/commcare_cloud/ansible/roles/couchdb2/files/vm.args.defaults +++ b/src/commcare_cloud/ansible/roles/couchdb2/files/vm.args.defaults @@ -3,3 +3,6 @@ # Set maximum SSL session lifetime to reap terminated replication readers -ssl session_lifetime 300 + +-kernel inet_dist_listen_min 9100 +-kernel inet_dist_listen_max 9200 From 046c319bc13554edec5453d3a2253408d85936fb Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Fri, 26 Sep 2025 16:12:51 +0530 Subject: [PATCH 14/25] add 3 new couch nodes that can be used replace existing nodes --- environments/backup-production/terraform.yml | 37 ++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/environments/backup-production/terraform.yml b/environments/backup-production/terraform.yml index fc012b187b..8d2d72a597 100644 --- a/environments/backup-production/terraform.yml +++ b/environments/backup-production/terraform.yml @@ -61,6 +61,43 @@ servers: group: "couchdb2" os: jammy + - server_name: "couch_a3-bk-production" + server_instance_type: t3a.large + network_tier: "db-private" + az: "c" + volume_size: 20 + volume_encrypted: yes + block_device: + volume_size: 1000 + encrypted: yes + group: "couchdb2" + os: jammy + + - server_name: "couch_a4-bk-production" + server_instance_type: t3a.small + network_tier: "db-private" + az: "c" + volume_size: 20 + volume_encrypted: yes + block_device: + volume_size: 10 + encrypted: yes + group: "couchdb2" + os: jammy + + - server_name: "couch_a5-bk-production" + server_instance_type: t3a.small + network_tier: "db-private" + az: "c" + volume_size: 20 + volume_encrypted: yes + block_device: + volume_size: 10 + encrypted: yes + group: "couchdb2" + os: jammy + + internal_albs: - name: "couch_alb-bk-production" listener_port: 25984 From 3370ca07e9f9395dfdba90bf6d3d6e48ecefc5e1 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Fri, 26 Sep 2025 16:55:46 +0530 Subject: [PATCH 15/25] update 3 new couch node information in the inventory files --- environments/backup-production/aws-resources.yml | 6 ++++++ environments/backup-production/inventory.ini | 9 +++++++++ environments/backup-production/inventory.ini.j2 | 6 ++++++ 3 files changed, 21 insertions(+) diff --git a/environments/backup-production/aws-resources.yml b/environments/backup-production/aws-resources.yml index 340da16a0d..baf933b0c8 100644 --- a/environments/backup-production/aws-resources.yml +++ b/environments/backup-production/aws-resources.yml @@ -6,6 +6,12 @@ couch_a1-bk-production: 10.212.41.62 couch_a1-bk-production.instance_id: i-0dde79ea855218d9f couch_a2-bk-production: 10.212.41.29 couch_a2-bk-production.instance_id: i-06d41c8d2726ca675 +couch_a3-bk-production: 10.212.41.164 +couch_a3-bk-production.instance_id: i-03a02355e6a300329 +couch_a4-bk-production: 10.212.41.77 +couch_a4-bk-production.instance_id: i-04b818049fe6b9aef +couch_a5-bk-production: 10.212.41.113 +couch_a5-bk-production.instance_id: i-09389818f56087850 couch_alb-bk-production: internal-couch-alb-bk-production-62333703.us-east-2.elb.amazonaws.com formplayer-efs: fs-0287582bff16a9a0b.efs.us-east-2.amazonaws.com frontend_alb-bk-production: internal-frontend-alb-bk-production-194015126.us-east-2.elb.amazonaws.com diff --git a/environments/backup-production/inventory.ini b/environments/backup-production/inventory.ini index e20cead792..423140882b 100644 --- a/environments/backup-production/inventory.ini +++ b/environments/backup-production/inventory.ini @@ -4,11 +4,20 @@ 10.212.41.62 hostname=couch-a1-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-0dde79ea855218d9f datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws [couch_a2] 10.212.41.29 hostname=couch-a2-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-06d41c8d2726ca675 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws +[couch_a3] +10.212.41.164 hostname=couch-a3-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-03a02355e6a300329 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws +[couch_a4] +10.212.41.77 hostname=couch-a4-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-04b818049fe6b9aef datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws +[couch_a5] +10.212.41.113 hostname=couch-a5-bk-production ufw_private_interface=ens5 ansible_python_interpreter=/usr/bin/python3 ec2_instance_id=i-09389818f56087850 datavol_device=/dev/sdf datavol_device1=/dev/sdf is_datavol_ebsnvme=yes root_encryption_mode=aws [couchdb2:children] couch_a0 couch_a1 couch_a2 +couch_a3 +couch_a4 +couch_a5 [couchdb2:vars] swap_size=8G diff --git a/environments/backup-production/inventory.ini.j2 b/environments/backup-production/inventory.ini.j2 index 854064f8da..d8c4fa068d 100644 --- a/environments/backup-production/inventory.ini.j2 +++ b/environments/backup-production/inventory.ini.j2 @@ -1,11 +1,17 @@ {{ __couch_a0__ }} {{ __couch_a1__ }} {{ __couch_a2__ }} +{{ __couch_a3__ }} +{{ __couch_a4__ }} +{{ __couch_a5__ }} [couchdb2:children] couch_a0 couch_a1 couch_a2 +couch_a3 +couch_a4 +couch_a5 [couchdb2:vars] swap_size=8G From 14380a70c61f251a03209a64161fa3b5ccd62040 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Fri, 26 Sep 2025 16:57:18 +0530 Subject: [PATCH 16/25] add step 1 form couch shard migrations --- .../migrations/0001-replace-all-nodes/step-1.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 environments/backup-production/migrations/0001-replace-all-nodes/step-1.yml diff --git a/environments/backup-production/migrations/0001-replace-all-nodes/step-1.yml b/environments/backup-production/migrations/0001-replace-all-nodes/step-1.yml new file mode 100644 index 0000000000..ec329a2e2a --- /dev/null +++ b/environments/backup-production/migrations/0001-replace-all-nodes/step-1.yml @@ -0,0 +1,2 @@ + target_allocation: + - couch_a0,couch_a2,couch_a3:3 \ No newline at end of file From a0cefa3133e66e787ebfa38b10d4ef650e32195e Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Fri, 26 Sep 2025 17:00:21 +0530 Subject: [PATCH 17/25] update known hosts file --- environments/backup-production/known_hosts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/environments/backup-production/known_hosts b/environments/backup-production/known_hosts index a3fabbe328..bbd7cfd891 100644 --- a/environments/backup-production/known_hosts +++ b/environments/backup-production/known_hosts @@ -1,12 +1,21 @@ 10.212.11.9 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBL8UvZUKCFRoUQkgsxBRR+tFYgpHP2m+Aj1cXB0B/zWmJRyVHqG3nW1Ag9C5M1dnClPk9LyClRnkZ3RTQXNI2RM= 10.212.11.9 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILBOPX2IitCfC8ZvzDuV3iDb89IzFqm2/eMQjHwSmSsJ 10.212.11.9 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDQBS6O4yHFXVhu+2t7iffcKDCu2h5WDYZt712YuLBmi+W3TmRrVfXM6FzoB7+REMB9MwRI2sM6Vt8kfbLvf5timrH1EP7lZY4oWvZuaLg1JtPAwIMk4icOLbQ23flgCD8zfKVLnqnmhqg/uXsJo/JKJFJOyWyGWnEbP4VbEDa/4ur6FT8kUZUGczF1ITpf4bKRXrd6MuIes4vyjy4789TrGIetpV3ipJ/QVvxHO7mdCh/AocznWnzgGBjSAsX9Hv24Jm1dX4zHowt6dALa2HQ9mjWoRJAAGXZHxCQzszh6ZWo9sw+0SoIlzOB3A5Gq72EK5MhJLhZzWRgZGkPOWf4gCri39Z11ZtK05W64YYHviDUyGaS+B8RAJSdE+A8dIFCn2qAor8Y0CPJ4t4M8nq7Av7obiCOdcdrePl2dyOXFA/aq4Ste8RtYfS9iu8QCUEfwViteTiLAX79V28Ox+waYYiq3/XHJd19pvaoFeYSgAN/pyZvZhb/VBDt+H0o0NZs= +10.212.41.113 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGAc8zrlcwfMrtzUoRQbGVilTb2efkmn9r/h0vgQxuLcTkK1fOwMRTRy7piEfhI7Wa9FQNN4gJonl6jDNdyMYLg= +10.212.41.113 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII0fKCXfLRx3l/WBvRzn2jsVZV1U8jNeBoxePVzNVQd2 +10.212.41.113 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCR6YqHpZh8znYZ5tECBDih6EqinSgtb8xgZf81ILniYxL43P4Oa4mXczKkq6q+UcECZqkcYd6imGt91ov+iiPj83kxzdnKPtAtBZHhzaUXX9mAlpSqgHNZGWnvvoMtmlUdXBWPJeU/LkWFIedvGpxpC7guv1gKAqc9J33wtJilqeAWwNMl0+g7764NHJ9F0mamfebjwTq+R3P8v0nTD7fdONKsYOwh0sCgDZbAI0MKLi8GzlM8LFIE3RTQlrl04jR7z+0vGq4HFF7n1ScJmT8r4sso5AgJgF6mRjxBLJzxsVyh5UqQJ3uXMWhVOGrYY8EtAzuDUL7WJwtNExaUasjs+yqIPkREhhaF9aZK0jjVyLoPOBmE0oajMHFcrnNOwTYZvB2wbdhROTHnSsNy8e72DJu5bOCCR5rviQTUwOmMLQ45kTUkvl4yVJV17igrNC9KQFFRpD4uYV1Wt39r6U0UkulzsdVqqCYfU/W9Ky4U2FrggXBJl7Pzrwdr5Wm07qs= 10.212.41.153 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBDCwyP1nSSs5iTbIwgRo+FhvW6HhXcrxuMOb/umz1q/WrC5ZXS+NOqrl90wv1paECjVhYBJr0DE2cqQEHV8wrs= 10.212.41.153 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA5aawps4jrRvaeP1WFYqFpvpyFHzTl8R8qM9iYi4Nh9 10.212.41.153 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCzffJ4c9rp4DLXwA3GfT5rJ5pPzQwCCLDr4nxDwdI476NA1VvOKo83b9IgCYHpIxritva5vgeKQM1AUxdUHSSEMSCQ+USySJLGm9LYgyuDuaw20p0nS4DIQL5AbhmhKgxeqblcEJVviLuakGzNlvn2dmLM7BWJjjtBh2/eQSIr2EXQH41Ik6SpF8SS9E0YM9BRjtfSZ9oppAtCmyyQ3aCxFOPne5S6XTPRrsncextKRBPQ0IJGR0EPVR32VLgaZB3BRxefNDTGFQ9mE9w3gFe3ojLkoBd0DbDaD96FwTAZHJUbtjEHDXoLU/IGgoAkCgQMPyybcGlmigFnVid/5pgleYWcf5vNS929YkyNvkZO09l0ykVy+3VqVnOvOepFxf/UTJFE5CzwT8oKYCzXuDhkexyTT0G2+yswe3r9gdN+JN5BMCzudo6Vpxd67jfaMaoA01HhSB6PYlxNHpgq6dpcZJTM1eZYEnzvF43OwH+z5BM7DK99pf6J2hjssW5GdPs= +10.212.41.164 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDZ1XDZi7BG8MOzB2nXIs9rV7VAxFwbkr+SolghYkEthMwXfKKh5+59+BkzSjbICsBE/icScKAAixgZzKHzPyA0= +10.212.41.164 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFmxdRf2fBX87X1d1hcHVvb8ASLqJ9heqr8l0wzhDwt2 +10.212.41.164 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/u+9HQ67WHo42D/NUxd7xl2ywG8cxx9qUVORHqFKWNap2+V8WHEOQx2QDoq9cuZEGYKmIqqiA/VcSoP11y8seYpoddcQps//gCvblCnfxJnQsSeyn9PF52/4eEbsxmZBA2gLRr6Hx8YFOg5qeJla/V+6vUWH3HvkxnmxE4MT+eZgBaLgmOQmn9+ClU5IJ/Mu9/eU5Xg3UWEo5a3aH0bIR+oYgdkzMRIBR0v+mwcy46BGbbCY0tmUSKfr9zSsQ1hUK+NYnpvnlwfM1PS6EOYx9vNCCSx1hG4ybVRw76qKNKkeaZ+8fW3CaC6+z3ufDPfVITiT5fvlkwa4/h4t8+kt6rclimQZyyc9Uyt+RJ/J1uIGrgwNsZhSBKD//wmsSJE5nXiP2J4zSwnaH8H9R1r/X2QUkfdi9oENqbKhFRICFimxCb93LxfdH3+Ky7YkNAYKCnfHE1vJVq3jRstUyjxpZPYtSKOr/nN1SRPxG9j4pU79k2pezbTURhx1n/byVaEs= 10.212.41.29 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBC2isCcPR33ML5bIYLXHMlIlkqJhipZ3ifDKwwbhPN1qNJD/mCSqjdhKbVK3Y40+ppCpp7J+6E3ycC142Yy+zyY= 10.212.41.29 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHM/4i0u5ouEkoGJqJRe6mNhx0Rlnr0OSFkCUYLEC5Un 10.212.41.29 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuVUS0GFVKj7Z/z3OUHV2W1+Spih8TIqQI2PHePt8pjcW+6A0Yki12aS461vySw63Wf1JJk4oaibHoKZ0rGN3qVWMw2/IJf1MO+ToVlamLWIVAR0cGfACJfsRKTdUVtRSnwuFUDkJ5la+54bxNse8XkPe51Pt0BMOQfpZEUpVpKMGmlEyR+jg0ej7chf6OK0w+u6n/pDB/VAbchcY7Q5XKnUSfKwZF+mLVRXqo4AWT9DAZHmEy3VbeSMOYJp62kopDamM3YRYpww9Zn3LoVgpFt5hHUAIl+10mGh1tvyjU2insmBEla2n4lHLW4+8CdkaypjPaEO4DwE7WS0nBrbDhODo7jaOCF1mZpZmO7UTuciLpCf6OdlIkqm3USW8MP3Wiav0wAMujJMEBpjPlX94ql+peXAU9AImPg3xQS92uYrP5MDRiVuIbzm8N5du1fNlyKwBBPu+/z/ERNPF89Q8vNLTFP7sZtwEl4yH2X0/LAG3pamV9HeMPOg7YC5A6FRE= 10.212.41.62 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFtVHkVi5ng4ViAxxnELn9L49Ws+xX3wJGZiwnmrfunGVRMJWxKUm/e132gHFmRuSX6ZnpmV0VTOGjHI9QvSk5k= 10.212.41.62 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFjUFthQ36Mm+E6Ztw31fA9YGKRlHJEFmOfVJ/Eja/Me -10.212.41.62 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCsGe1eSjvQiSxtczhaoCfTkmh0/yQNx6/z6ISBao1xkCA32ti7oAdZxsDRb9Ehm9/Z/HBWdUoTJg05fSLdxXLPCyP7YjksMT8kdk2ODimYTyVKTcoFAkrVbIYiKEMAh6a8Un6HwOc6Kl4bUuohUAxVJhE0rEHlhH1nAng0eiLJhJn3rKxG2j2MwAu+Kp9dv5/eQAtEnIUhW1iFU2uOA+eFjxXze/v25Iia1qOxNuaOLnUsDYyNDDn1JXZdSM2iyvC+Bg1/Ffb2GOJ5iiPOUe4P/ZHmZwJBuTgl0b7cd8m3ic/K9ZPM6ZbewkfowS+B5ZV2YAPw3zT3dS62xQtrFM7YpdISdYFTVleChwdv1ErzX/bumDSc/KyFF/ZCCR2XvNQ+Lv6qNFKiqJEUpCM6hi+SsAudbl00As5hH02jJ9icjgHBsa2PnJsnfOAZoIhJBNSuS1quleltFfMGjmcY/R4K/oqKQkgRQSwz7iyz/r9Otd7s0vvUJ9NxcVvg+uVcBS0= \ No newline at end of file +10.212.41.62 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCsGe1eSjvQiSxtczhaoCfTkmh0/yQNx6/z6ISBao1xkCA32ti7oAdZxsDRb9Ehm9/Z/HBWdUoTJg05fSLdxXLPCyP7YjksMT8kdk2ODimYTyVKTcoFAkrVbIYiKEMAh6a8Un6HwOc6Kl4bUuohUAxVJhE0rEHlhH1nAng0eiLJhJn3rKxG2j2MwAu+Kp9dv5/eQAtEnIUhW1iFU2uOA+eFjxXze/v25Iia1qOxNuaOLnUsDYyNDDn1JXZdSM2iyvC+Bg1/Ffb2GOJ5iiPOUe4P/ZHmZwJBuTgl0b7cd8m3ic/K9ZPM6ZbewkfowS+B5ZV2YAPw3zT3dS62xQtrFM7YpdISdYFTVleChwdv1ErzX/bumDSc/KyFF/ZCCR2XvNQ+Lv6qNFKiqJEUpCM6hi+SsAudbl00As5hH02jJ9icjgHBsa2PnJsnfOAZoIhJBNSuS1quleltFfMGjmcY/R4K/oqKQkgRQSwz7iyz/r9Otd7s0vvUJ9NxcVvg+uVcBS0= +10.212.41.77 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFYND/Zq8wCaFloS3WomxtpWNfrxt8k01WJd9SwupiPWH8Y04/BXN+xTmIMFtJ03o3KwtC0Jsc2m4XKW4VOM7vo= +10.212.41.77 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAALXy9GsUokb0IEob0LRM/wdu21oJ9vYgD1UJ8qLSb3 +10.212.41.77 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDWY05w+AsyuEIo+bDGthXEu+r8pYC4iviC+DtG4cq4cKOc0c6Vst5viOZ7RnZLPksDwzLsntJlK/KFzShHbCk007smNoB4vporQl9A6cA98tBotRwj1g4BKpvYU4yeimx0QQRJ6ZeV0s9OwCOZ15tuWauRpf2vq33XiQEOl6pi6/sBN431/2LVReadxKgFreivKpFlNj4dJtd7S+9yTgtW/dm509reDll1VC+nivpYMc7TfbDE4Vg+9Jc84s8aAQQ/P9+ay+6mxPNikyUn/zHU0gO0zMRYDHZBAcoa4HansI/LLyJfSxHphyZPU4YRNNYQYSMr8wByCBXbLaaNAdaAu2oC60lSYHP1shDe+1kJjNt4R7/a4CQ7pbGQaktlMQ0/nmuVC4+UvkwO4hunvzhyX+w+oVyLdfR5FGl/gFZeJK4a/ESs1kTWLcRez+jTzeaK7F8eyzQF5J4VMX17XYl6rbBIe0yyXPDRtYDg3YB5OkMeio/ktlESM++Ok1wlnas= \ No newline at end of file From 753c5b4a3f150494caa8393d4d01439c4b0d489e Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Fri, 26 Sep 2025 19:57:21 +0530 Subject: [PATCH 18/25] add all the steps for migration process --- .../migrations/0001-replace-all-nodes/step-2.yml | 2 ++ .../migrations/0001-replace-all-nodes/step-3.yml | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 environments/backup-production/migrations/0001-replace-all-nodes/step-2.yml create mode 100644 environments/backup-production/migrations/0001-replace-all-nodes/step-3.yml diff --git a/environments/backup-production/migrations/0001-replace-all-nodes/step-2.yml b/environments/backup-production/migrations/0001-replace-all-nodes/step-2.yml new file mode 100644 index 0000000000..006ac4bc43 --- /dev/null +++ b/environments/backup-production/migrations/0001-replace-all-nodes/step-2.yml @@ -0,0 +1,2 @@ +target_allocation: + - couch_a2,couch_a3,couch_a4:3 \ No newline at end of file diff --git a/environments/backup-production/migrations/0001-replace-all-nodes/step-3.yml b/environments/backup-production/migrations/0001-replace-all-nodes/step-3.yml new file mode 100644 index 0000000000..f34a4fe8da --- /dev/null +++ b/environments/backup-production/migrations/0001-replace-all-nodes/step-3.yml @@ -0,0 +1,2 @@ +target_allocation: + - couch_a3,couch_a4,couch_a5:3 \ No newline at end of file From 48113c328719dd387fc0d29088c7e19f8e08a615 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Fri, 26 Sep 2025 20:06:31 +0530 Subject: [PATCH 19/25] remove space from yml file --- .../migrations/0001-replace-all-nodes/step-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/backup-production/migrations/0001-replace-all-nodes/step-1.yml b/environments/backup-production/migrations/0001-replace-all-nodes/step-1.yml index ec329a2e2a..7c229cb7c3 100644 --- a/environments/backup-production/migrations/0001-replace-all-nodes/step-1.yml +++ b/environments/backup-production/migrations/0001-replace-all-nodes/step-1.yml @@ -1,2 +1,2 @@ - target_allocation: +target_allocation: - couch_a0,couch_a2,couch_a3:3 \ No newline at end of file From c01b307a7ee3281670b5ad7d0732cd0c77be42ff Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Tue, 30 Sep 2025 20:37:46 +0530 Subject: [PATCH 20/25] Refactor CouchDB service handling to use Ansible variable for service name based on Ubuntu version --- src/commcare_cloud/commands/migrations/couchdb.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/commcare_cloud/commands/migrations/couchdb.py b/src/commcare_cloud/commands/migrations/couchdb.py index 2e23b2db9d..2549985618 100644 --- a/src/commcare_cloud/commands/migrations/couchdb.py +++ b/src/commcare_cloud/commands/migrations/couchdb.py @@ -456,7 +456,9 @@ def start_stop_service(ansible_context, service_state, check_mode=False): if check_mode: extra_args.append('--check') - for service in ('monit', 'couchdb2'): + # Use Ansible variable 'couchdb_service_name' which resolves per-host based on Ubuntu version + # Ubuntu 18.04 uses 'couchdb2', Ubuntu 22.04+ uses 'couchdb' + for service in ('monit', "{{ couchdb_service_name }}"): args = 'name={} state={}'.format(service, service_state) run_ansible_module(ansible_context, 'couchdb2', 'service', args, extra_args=extra_args) From 91657425ebe7e38de2ce46d3654e2b8059d81d0d Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Tue, 30 Sep 2025 22:05:15 +0530 Subject: [PATCH 21/25] Add logic to delete SSH key files after authorized_key removals in the auth playbook to fix race conditions --- src/commcare_cloud/commands/migrations/copy_files.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/commcare_cloud/commands/migrations/copy_files.py b/src/commcare_cloud/commands/migrations/copy_files.py index 3df7691b63..c2b3b29d42 100644 --- a/src/commcare_cloud/commands/migrations/copy_files.py +++ b/src/commcare_cloud/commands/migrations/copy_files.py @@ -277,7 +277,7 @@ def _run_auth_playbook(plan, ansible_context, action, working_directory): auth_pairs.update({ (target_host, config.source_host, config.source_user) for config in source_configs }) - + keys_to_delete = set() for target_host, source_host, source_user in auth_pairs: _set_auth_key( AnsibleContext(None, plan.source_env), @@ -288,6 +288,13 @@ def _run_auth_playbook(plan, ansible_context, action, working_directory): working_directory, remove=(action == 'remove'), ) + if action == 'remove': + keys_to_delete.add(os.path.join(working_directory, 'id_rsa_{}.pub'.format(target_host))) + + # Delete key files only after all authorized_key removals are complete + for key_file in keys_to_delete: + if os.path.exists(key_file): + os.remove(key_file) def _set_auth_key(source_context, source_host, source_user, @@ -303,6 +310,3 @@ def _set_auth_key(source_context, source_host, source_user, state = 'absent' if remove else 'present' args = "user={} state={} key={{{{ lookup('file', '{}') }}}}".format(source_user, state, key_path) run_ansible_module(source_context, source_host, 'authorized_key', args) - - if remove: - os.remove(key_path) From 3d47b07c787c13466b65a7c7a357a34ae962d46b Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Fri, 3 Oct 2025 19:49:16 +0530 Subject: [PATCH 22/25] Update server instance type and block device volume size in Terraform configuration for backup production environment --- environments/backup-production/terraform.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environments/backup-production/terraform.yml b/environments/backup-production/terraform.yml index 8d2d72a597..4851162c3d 100644 --- a/environments/backup-production/terraform.yml +++ b/environments/backup-production/terraform.yml @@ -74,13 +74,13 @@ servers: os: jammy - server_name: "couch_a4-bk-production" - server_instance_type: t3a.small + server_instance_type: t3a.large network_tier: "db-private" az: "c" volume_size: 20 volume_encrypted: yes block_device: - volume_size: 10 + volume_size: 1000 encrypted: yes group: "couchdb2" os: jammy From bea6586f06982f69ca5b471d7284c56ad4973736 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Mon, 6 Oct 2025 15:11:54 +0530 Subject: [PATCH 23/25] update instnace type of couch_a5 and add more storage --- environments/backup-production/terraform.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/environments/backup-production/terraform.yml b/environments/backup-production/terraform.yml index 4851162c3d..3b0a668c2d 100644 --- a/environments/backup-production/terraform.yml +++ b/environments/backup-production/terraform.yml @@ -86,13 +86,13 @@ servers: os: jammy - server_name: "couch_a5-bk-production" - server_instance_type: t3a.small + server_instance_type: t3a.large network_tier: "db-private" az: "c" volume_size: 20 volume_encrypted: yes block_device: - volume_size: 10 + volume_size: 1000 encrypted: yes group: "couchdb2" os: jammy @@ -104,5 +104,5 @@ internal_albs: target_port: 15984 health_check_interval: 30 targets: - - couch_a0-bk-production + - couch_a3-bk-production From 1d476f410e0a196e234cf726599bb4b457fe91c5 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Mon, 6 Oct 2025 19:55:30 +0530 Subject: [PATCH 24/25] add couch nodes in alb --- environments/backup-production/terraform.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/environments/backup-production/terraform.yml b/environments/backup-production/terraform.yml index 3b0a668c2d..6d01312bc3 100644 --- a/environments/backup-production/terraform.yml +++ b/environments/backup-production/terraform.yml @@ -105,4 +105,6 @@ internal_albs: health_check_interval: 30 targets: - couch_a3-bk-production + - couch_a4-bk-production + - couch_a5-bk-production From 97ae9989b7ebc0872e1f8f0cfc1337b0a3d33256 Mon Sep 17 00:00:00 2001 From: Amit Phulera Date: Wed, 29 Oct 2025 12:00:40 +0530 Subject: [PATCH 25/25] remove older nodes from terraform config --- environments/backup-production/terraform.yml | 43 +++----------------- 1 file changed, 5 insertions(+), 38 deletions(-) diff --git a/environments/backup-production/terraform.yml b/environments/backup-production/terraform.yml index 17bf39cb1a..251ba8313f 100644 --- a/environments/backup-production/terraform.yml +++ b/environments/backup-production/terraform.yml @@ -1,5 +1,5 @@ account_alias: commcare-backup-production -terraform_version: '1.5' +terraform_version: '1.13' state_bucket: dimagi-terraform state_bucket_region: "us-east-1" region: "us-east-2" @@ -7,11 +7,13 @@ environment: "bk-production" skip_ebs_snapshots: "yes" ssl_policy: 'ELBSecurityPolicy-FS-1-2-Res-2020-10' azs: - - "us-east-2a" + - "us-east-2c" - "us-east-2b" + - "us-east-2a" az_codes: - - a + - c - b + - a vpc_begin_range: "10.212" ec2_metadata_tokens_required: yes @@ -25,41 +27,6 @@ servers: group: "control" os: jammy - - server_name: "couch_a0-bk-production" - server_instance_type: t3a.large - network_tier: "db-private" - az: "a" - volume_size: 20 - volume_encrypted: yes - block_device: - volume_size: 1000 - encrypted: yes - group: "couchdb2" - os: jammy - - - server_name: "couch_a1-bk-production" - server_instance_type: t3a.large - network_tier: "db-private" - az: "c" - volume_size: 20 - volume_encrypted: yes - block_device: - volume_size: 1000 - encrypted: yes - group: "couchdb2" - os: jammy - - - server_name: "couch_a2-bk-production" - server_instance_type: t3a.large - network_tier: "db-private" - az: "c" - volume_size: 20 - volume_encrypted: yes - block_device: - volume_size: 1000 - encrypted: yes - group: "couchdb2" - os: jammy - server_name: "couch_a3-bk-production" server_instance_type: t3a.large