diff --git a/reproducer.yml b/reproducer.yml index f5433b3544..43d5539983 100644 --- a/reproducer.yml +++ b/reproducer.yml @@ -108,18 +108,22 @@ name: firewalld state: restarted + # NOTE(dpawlik): Keep support for legacy workflow where + # controller host is created and actions are done not on + # hypervisor directly, but on controller. # NOTE(dpawlik): Synchronize current Zuul repositories stored in $HOME/src # to controller-0 to make sure all projects are in state as they should. - - name: Synchronize src dir with controller-0 + # FIXME: that value should be available from playbooks/uni/run.yml + - name: Set hypervisor name if not defined + when: hypervisor is not defined + ansible.builtin.set_fact: + hypervisor: "{{ ansible_fqdn }}" + - name: Synchronize src dir when: - not cifmw_deploy_reproducer_env | default(true) | bool - vars: - sync_dir: "{{ ansible_user_dir }}/src" - ansible.posix.synchronize: - src: "{{ sync_dir }}/" - dest: "zuul@controller-0:{{ sync_dir }}" - archive: true - recursive: true + ansible.builtin.include_role: + name: reproducer + tasks_from: sync_src_dir.yml # NOTE(dpawlik): After calling reproducer role using ZIronic tool, # when the bootstrap phase has been completed, it generates a file @@ -133,6 +137,28 @@ ansible.builtin.include_role: name: reproducer tasks_from: overwrite_zuul_vars.yml + + # NOTE(dpawlik): Since we use ZIronic, some variables are not + # redirected to nested ansible execution - they needs to be + # included on executing host - controller-0. + - name: Compute additional deploy_architecture_args with secrets + when: + - not cifmw_deploy_reproducer_env | default(true) | bool + vars: + basic_secret_files: + - "{{ ansible_user_dir }}/secrets/registry_token_creds.yaml" + ansible.builtin.include_role: + name: reproducer + tasks_from: compute_additional_args.yml + loop: "{{ cifmw_deploy_architecture_secret_files | default(basic_secret_files) }}" + loop_control: + loop_var: secret_file + + - name: Print final cifmw_deploy_architecture_args + ansible.builtin.debug: + msg: > + Current cifmw_deploy_architecture_args {{ cifmw_deploy_architecture_args | default('') }} + - name: Run deployment if instructed to when: - cifmw_deploy_architecture | default(false) | bool @@ -141,7 +167,10 @@ poll: 20 delegate_to: controller-0 ansible.builtin.command: - cmd: "$HOME/deploy-architecture.sh {{ cifmw_deploy_architecture_args | default('') }}" + cmd: > + $HOME/deploy-architecture.sh + -e hypervisor="{{ hypervisor }}" + {{ cifmw_deploy_architecture_args | default('') }} - name: Run post deployment if instructed to when: diff --git a/roles/reproducer/files/merge_yaml_override.py b/roles/reproducer/files/merge_yaml_override.py new file mode 100644 index 0000000000..c1c9d30f64 --- /dev/null +++ b/roles/reproducer/files/merge_yaml_override.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Merge two YAML files whose root is a mapping: keys from OVERRIDE replace BASE +# recursively for nested mappings; non-dict values (scalars, lists) are +# replaced entirely by OVERRIDE. Requires PyYAML (python3-pyyaml / python3-yaml +# on RPM systems). + +from __future__ import annotations + +import sys + +try: + import yaml +except ImportError: + sys.stderr.write( + "merge_yaml_override.py: PyYAML is required " + "(e.g. dnf install python3-pyyaml or pip install pyyaml)\n") + sys.exit(1) + + +def deep_merge(base: object, override: object) -> object: + if base is None: + return override + if override is None: + return base + if isinstance(base, dict) and isinstance(override, dict): + out = dict(base) + for key, val in override.items(): + if key in out: + out[key] = deep_merge(out[key], val) + else: + out[key] = val + return out + return override + + +def main() -> None: + argc = len(sys.argv) + if argc not in (3, 4): + sys.stderr.write( + f"usage: {sys.argv[0]} FILE1_BASE.yml FILE2_OVERRIDE.yml [MERGED_OUT.yml]\n" + " Deep-merge two YAML mappings: FILE2 wins on duplicate keys (nested dicts merged).\n" + " Scalars and lists from FILE2 replace FILE1. If MERGED_OUT is omitted, print to stdout.\n" + ) + sys.exit(2) + + base_path, override_path = sys.argv[1], sys.argv[2] + out_path = sys.argv[3] if argc == 4 else None + + with open(base_path, encoding="utf-8") as f: + base = yaml.safe_load(f) + with open(override_path, encoding="utf-8") as f: + override = yaml.safe_load(f) + + if base is None: + base = {} + if override is None: + override = {} + + if not isinstance(base, dict): + sys.stderr.write( + f"{base_path}: root must be a mapping, got {type(base).__name__}\n" + ) + sys.exit(3) + if not isinstance(override, dict): + sys.stderr.write( + f"{override_path}: root must be a mapping, got {type(override).__name__}\n" + ) + sys.exit(3) + + merged = deep_merge(base, override) + dump_kw = dict( + default_flow_style=False, + sort_keys=False, + allow_unicode=True, + ) + if out_path is None: + yaml.safe_dump(merged, sys.stdout, **dump_kw) + else: + with open(out_path, "w", encoding="utf-8") as out_f: + yaml.safe_dump(merged, out_f, **dump_kw) + + +if __name__ == "__main__": + main() diff --git a/roles/reproducer/tasks/compute_additional_args.yml b/roles/reproducer/tasks/compute_additional_args.yml new file mode 100644 index 0000000000..bcae05d166 --- /dev/null +++ b/roles/reproducer/tasks/compute_additional_args.yml @@ -0,0 +1,12 @@ +--- +- name: "Checking if file exists {{ secret_file }}" + ansible.builtin.stat: + path: "{{ secret_file }}" + register: _current_file + +- name: Add secret file into cifmw_deploy_architecture_args when exists + when: _current_file.stat.exists + ansible.builtin.set_fact: + cifmw_deploy_architecture_args: > + {{ cifmw_deploy_architecture_args | default('') }} + -e @{{ secret_file }} diff --git a/roles/reproducer/tasks/overwrite_zuul_vars.yml b/roles/reproducer/tasks/overwrite_zuul_vars.yml index 8cf458ce29..e9158489a6 100644 --- a/roles/reproducer/tasks/overwrite_zuul_vars.yml +++ b/roles/reproducer/tasks/overwrite_zuul_vars.yml @@ -15,46 +15,58 @@ - name: Overwrite reproducer-variables.yml when ZIronic used when: _current_zuul_vars.stat.exists + vars: + temp_reproducer_var_path: /tmp/reproducer-variables.yml + temp_merged_reproducer_var_path: /tmp/merged-reproducer-variables.yml block: - - name: Dump reproducer-variables.yml to var + - name: Slurp reproducer-variables.yml to hypervisor ansible.builtin.slurp: src: "{{ cifmw_basedir }}/parameters/reproducer-variables.yml" register: reproducer_original delegate_to: controller-0 no_log: "{{ cifmw_nolog | default(true) | bool }}" - - name: Dump zuul_vars.yaml to var - ansible.builtin.slurp: - src: "{{ ansible_user_dir }}/configs/zuul_vars.yaml" - register: zuul_job_vars + - name: Create temp file reproducer-variables.yml on hypervisor + ansible.builtin.copy: + content: "{{ reproducer_original.content | b64decode }}" + dest: "{{ temp_reproducer_var_path }}" + mode: "0664" no_log: "{{ cifmw_nolog | default(true) | bool }}" - - name: Load current job zuul_vars.yaml - ansible.builtin.include_vars: - file: "{{ ansible_user_dir }}/configs/zuul_vars.yaml" - name: _current_zuul_vars_include + - name: Copy merge yamls script + become: true + ansible.builtin.copy: + src: merge_yaml_override.py + dest: /usr/local/bin/merge_yaml_override + mode: "0755" - - name: Decode reproducer-variables.yml content - ansible.builtin.set_fact: - reproducer_vars_content: "{{ reproducer_original.content | b64decode | from_yaml }}" + - name: Execute merge script + ansible.builtin.shell: > + python3 /usr/local/bin/merge_yaml_override + {{ temp_reproducer_var_path }} + {{ ansible_user_dir }}/configs/zuul_vars.yaml > + {{ temp_merged_reproducer_var_path }} no_log: "{{ cifmw_nolog | default(true) | bool }}" - - name: Filter zuul vars to only keys present in reproducer-variables - ansible.builtin.set_fact: - _zuul_vars_filtered: >- - {{ - _current_zuul_vars_include | dict2items - | selectattr('key', 'in', reproducer_vars_content.keys()) - | items2dict - }} + - name: Slurp merged reproducer-variables.yml from hypervisor + ansible.builtin.slurp: + src: "{{ temp_merged_reproducer_var_path }}" + register: merged_reproducer_slurp no_log: "{{ cifmw_nolog | default(true) | bool }}" - name: Write back merged reproducer-variables.yml ansible.builtin.copy: - content: >- - {{ reproducer_vars_content | combine(_zuul_vars_filtered, recursive=true) | to_nice_yaml }} + content: "{{ merged_reproducer_slurp.content | b64decode }}" dest: "{{ cifmw_basedir }}/parameters/reproducer-variables.yml" - mode: '0664' + mode: "0664" backup: true no_log: "{{ cifmw_nolog | default(true) | bool }}" delegate_to: controller-0 + + - name: Overwrite custom-params.yml + ansible.builtin.copy: + content: "{{ merged_reproducer_slurp.content | b64decode }}" + dest: "{{ cifmw_basedir }}/artifacts/parameters/custom-params.yml" + mode: "0664" + no_log: "{{ cifmw_nolog | default(true) | bool }}" + delegate_to: controller-0 diff --git a/roles/reproducer/tasks/sync_src_dir.yml b/roles/reproducer/tasks/sync_src_dir.yml new file mode 100644 index 0000000000..6790407152 --- /dev/null +++ b/roles/reproducer/tasks/sync_src_dir.yml @@ -0,0 +1,64 @@ +--- +# NOTE(dpawlik): Synchronize data when ZIronic provision+bootstrap the +# hypervisor AND Zuul job uses ZIronic flavor which is executing +# job directly on hypervisor. When job starts, src and secrets dir are "recreated" +# by zuul-executor with current job data. Sync the data with +# controller-0 which is executing nested Ansible. +- name: Synchronize hypervisor secrets dir with controller-0 + vars: + sync_dir: "{{ ansible_user_dir }}/secrets" + ansible.posix.synchronize: + src: "{{ sync_dir }}/" + dest: "zuul@controller-0:{{ sync_dir }}" + archive: true + recursive: true + no_log: "{{ cifmw_nolog | default(true) | bool }}" + +- name: Synchronize hypervisor src dir with controller-0 + vars: + sync_dir: "{{ ansible_user_dir }}/src" + ansible.posix.synchronize: + src: "{{ sync_dir }}/" + dest: "zuul@controller-0:{{ sync_dir }}" + archive: true + recursive: true + no_log: "{{ cifmw_nolog | default(true) | bool }}" + +# NOTE(dpawlik): Synchronize data when ZIronic provision+bootstrap the +# hypervisor BUT legacy workflow is used (spawning controller VM +# which is executing all other playbooks). When job starts, all +# data like secrets, src dir etc. is stored on controller VM, but because +# later tasks are using nested ansible execution, then all playbooks needs +# to be up-to-date. +# That case would be mostly used by developers. Synchronize module +# would ignore if sync would happen to the same host. No need to synchronize +# with controller-0 again, due tasks above would do that. +- name: Synchronize controller VM configs dir with hypervisor + vars: + sync_dir: "{{ ansible_user_dir }}/configs" + ansible.posix.synchronize: + src: "{{ sync_dir }}/" + dest: "zuul@{{ hypervisor }}:{{ sync_dir }}" + archive: true + recursive: true + no_log: "{{ cifmw_nolog | default(true) | bool }}" + +- name: Synchronize controller secrets dir with hypervisor + vars: + sync_dir: "{{ ansible_user_dir }}/secrets" + ansible.posix.synchronize: + src: "{{ sync_dir }}/" + dest: "zuul@{{ hypervisor }}:{{ sync_dir }}" + archive: true + recursive: true + no_log: "{{ cifmw_nolog | default(true) | bool }}" + +- name: Synchronize src dir with hypervisor + vars: + sync_dir: "{{ ansible_user_dir }}/src" + ansible.posix.synchronize: + src: "{{ sync_dir }}/" + dest: "zuul@{{ hypervisor }}:{{ sync_dir }}" + archive: true + recursive: true + no_log: "{{ cifmw_nolog | default(true) | bool }}"