diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index 19d303df5..35a639aa3 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -2,6 +2,7 @@ # ex: set filetype=python: import os +from dataclasses import dataclass from buildbot.plugins import steps, util, worker from buildbot.steps.shell import Test @@ -9,6 +10,7 @@ from constants import BUILDERS_INSTALL, OS_INFO from master_common import base_master_config from utils import canStartBuild, envFromProperties, getScript, nextBuild, printEnv + cfg_dir = os.path.abspath(os.path.dirname(__file__)) # Non autogen master. For now the directory structure is: @@ -33,118 +35,81 @@ c = BuildmasterConfig = base_master_config(config) artifactsURL = os.environ["ARTIFACTS_URL"] -####### UTILS -def getRpmUpgradeStep(): +INSTALL_UPGRADE_PROPERTIES_LIST = [ + # Must be set to True in BB for upgrade scripts to work + "BB_CI", + # TODO(cvicentiu) compared to arm64 instead of aarch64 in scripts/deb-install.sh + # amd64, aarch64, ppc64le, s390x, x86 + "arch", + # Where artifacts are downloaded from ci.mariadb.org + "artifactsURL", + # Impacts upgrade and install logic. Same value as the branch in git + # (incl. bb- prefix and other suffixes). + "branch", + # almalinux, centos, debian, fedora, openeuler, rhel, ubuntu, rockylinux + "dist_name", + # compared to "yes" in rpm-upgrade.sh, but never set. + # TODO(cvicentiu) investigate and fix this. + "is_main_tree", + # TODO(cvicentiu) This should be handled within the script, not BB, + # mariadb_version should be enough. + # Extracted from mariadb_version variable as a separate BB step. + "major_version", + # Eg. mariadb-10.11.12, comes from parent (tarball) builder props, + # based on tarball filename. + "mariadb_version", + # Set from parent (tarball) builder. First two numbers from mariadb_version. + "master_branch", + # TODO(cvicentiu) investigate and remove this one. + # Set to "yes" / "no" within this file, but never used anywhere. + "needsGalera", + # Set to the builder that triggered this install/upgrade + # Used during install scripts to create builder URL. + "parentbuildername", + # TODO(cvicentiu) This should be removed from install / upgrade scripts. + # and finally removed from here. + # Always set to "yes" within this file. + "systemdCapability", + # Set by tarball docker. + "tarbuildnum", + # "server", "all", "columnstore", "deps" + # Impacts which tests are run. Set within this file only. + "test_mode", + # "major", "minor" + # Impacts which tests are run. set within this file. + "test_type", + # Loaded from os_info.yaml. Impacts upgrade & install tests. + "version_name", +] + + +def _get_test_step(name: str, script: str): return Test( - name="upgrade", + name=name, haltOnFailure=True, - description=["testing", "upgrade"], - descriptionDone=["test", "upgrade"], - env=envFromProperties( - [ - "BB_CI", - "arch", - "artifactsURL", - "branch", - "dist_name", - "is_main_tree", - "major_version", - "mariadb_version", - "master_branch", - "needsGalera", - "parentbuildername", - "systemdCapability", - "test_mode", - "test_type", - "version_name", - ] - ), - command=["./rpm-upgrade.sh"], + description=["testing", name], + descriptionDone=["test", name], + env=envFromProperties(INSTALL_UPGRADE_PROPERTIES_LIST), + command=[script], + ) +####### UTILS +def getRpmUpgradeStep(): + return _get_test_step(name="upgrade", script="./rpm-upgrade.sh") + + def getRpmInstallStep(): - return Test( - name="install", - haltOnFailure=True, - description=["testing", "install"], - descriptionDone=["test", "install"], - env=envFromProperties( - [ - "BB_CI", - "arch", - "artifactsURL", - "branch", - "dist_name", - "major_version", - "mariadb_version", - "master_branch", - "needsGalera", - "parentbuildername", - "systemdCapability", - "test_mode", - "test_type", - "version_name", - ] - ), - command=["./rpm-install.sh"], - ) + return _get_test_step(name="install", script="./rpm-install.sh") def getDebUpgradeStep(): - return Test( - name="upgrade", - haltOnFailure=True, - description=["testing", "upgrade"], - descriptionDone=["test", "upgrade"], - env=envFromProperties( - [ - "BB_CI", - "arch", - "artifactsURL", - "branch", - "dist_name", - "major_version", - "mariadb_version", - "master_branch", - "needsGalera", - "parentbuildername", - "systemdCapability", - "test_mode", - "test_type", - "version_name", - ] - ), - command=["./deb-upgrade.sh"], - ) + return _get_test_step(name="upgrade", script="./deb-upgrade.sh") def getDebInstallStep(): - return Test( - name="install", - haltOnFailure=True, - description=["testing", "install"], - descriptionDone=["test", "install"], - env=envFromProperties( - [ - "BB_CI", - "arch", - "artifactsURL", - "branch", - "dist_name", - "major_version", - "mariadb_version", - "master_branch", - "needsGalera", - "parentbuildername", - "systemdCapability", - "test_mode", - "test_type", - "version_name", - ] - ), - command=["./deb-install.sh"], - ) + return _get_test_step(name="install", script="./deb-install.sh") def getMajorVersionStep(): @@ -208,6 +173,33 @@ f_rpm_upgrade.addStep(getRpmUpgradeStep()) c["workers"] = [] c["builders"] = [] + +@dataclass +class LibVirtWorkerSpecs: + platform: str + os_name: str + os_version: str + + def __post_init__(self): + self.host, self.url = config["private"]["libvirt_workers"][platform] + + @property + def name(self) -> str: + return f"{self.host}-{self.os_name}-{self.os_version}-{self.platform}" + + @property + def connection_url(self) -> str: + return self.url + + @property + def image_path(self) -> str: + return f"/var/libvirt/images/{self.name}", + + @property + def password(self) -> str: + return config["private"]["worker_pass"]["libvirt"] + + # Add the workers and builds based on the configured install builders (see constants.py) for builder_name in BUILDERS_INSTALL: # Parse builder name @@ -217,50 +209,28 @@ for builder_name in BUILDERS_INSTALL: os_info_name = os_name + "-" + os_version - libvirt_worker_name = ( - config["private"]["libvirt_workers"][platform][0] - + "-" - + os_name - + "-" - + os_version - + "-" - + platform - ) - connection_url = config["private"]["libvirt_workers"][platform][1] - image_path = "/var/libvirt/images/" + libvirt_worker_name + worker_specs = LibVirtWorkerSpecs(platform=platform, + os_name=os_name, + os_version=os_version) c["workers"].append( worker.LibVirtWorker( - libvirt_worker_name, - config["private"]["worker_pass"]["libvirt"], - util.Connection(connection_url), - image_path, + worker_specs.name, + worker_specs.password, + util.Connection(worker_specs.connection_url), + worker_specs.image_path, build_wait_timeout=0, max_builds=1, ) ) - if builder_type == "deb": - factory_install = f_deb_install - factory_upgrade = f_deb_upgrade - build_arch = platform - elif builder_type == "rpm": - factory_install = f_rpm_install - factory_upgrade = f_rpm_upgrade - build_arch = ( - os_name + str(OS_INFO[os_info_name]["version_name"]) + "-" + platform - ) - - # FIXME - all RPM's should follow the same conventions! - if os_name == "centos" and OS_INFO[os_info_name]["version_name"] >= 9: - if platform == "amd64": - platform = "x86_64" - build_arch = f"centos/{OS_INFO[os_info_name]['version_name']}/{platform}" + factory_install = f_deb_install if builder_type == "deb" else f_rpm_install + factory_upgrade = f_deb_upgrade if builder_type == "deb" else f_rpm_upgrade c["builders"].append( util.BuilderConfig( name=builder_name, - workernames=libvirt_worker_name, + workernames=worker_specs.name, tags=[os_name, builder_type, "install", "kvm"], collapseRequests=True, nextBuild=nextBuild, @@ -270,7 +240,7 @@ for builder_name in BUILDERS_INSTALL: "needsGalera": "yes", "dist_name": os_name, "version_name": OS_INFO[os_info_name]["version_name"], - "arch": build_arch, + "arch": platform, "BB_CI": True, "artifactsURL": artifactsURL, }, @@ -284,7 +254,7 @@ for builder_name in BUILDERS_INSTALL: c["builders"].append( util.BuilderConfig( name=major_upgrade_name, - workernames=libvirt_worker_name, + workernames=worker_specs.name, tags=[os_name, builder_type, "upgrade", "kvm", "major", "server"], collapseRequests=True, nextBuild=nextBuild, @@ -294,7 +264,7 @@ for builder_name in BUILDERS_INSTALL: "needsGalera": "yes", "dist_name": os_name, "version_name": OS_INFO[os_info_name]["version_name"], - "arch": build_arch, + "arch": platform, "test_mode": "server", "test_type": "major", "BB_CI": True, @@ -310,7 +280,7 @@ for builder_name in BUILDERS_INSTALL: c["builders"].append( util.BuilderConfig( name=minor_upgrade_name + "-all", - workernames=libvirt_worker_name, + workernames=worker_specs.name, tags=[os_name, builder_type, "upgrade", "kvm", "minor", "all"], collapseRequests=True, nextBuild=nextBuild, @@ -320,7 +290,7 @@ for builder_name in BUILDERS_INSTALL: "needsGalera": "yes", "dist_name": os_name, "version_name": OS_INFO[os_info_name]["version_name"], - "arch": build_arch, + "arch": platform, "test_mode": "all", "test_type": "minor", "BB_CI": True, @@ -334,7 +304,7 @@ for builder_name in BUILDERS_INSTALL: c["builders"].append( util.BuilderConfig( name=minor_upgrade_name + "-columnstore", - workernames=libvirt_worker_name, + workernames=worker_specs.name, tags=[os_name, builder_type, "upgrade", "kvm", "minor", "columnstore"], collapseRequests=True, nextBuild=nextBuild, @@ -344,7 +314,7 @@ for builder_name in BUILDERS_INSTALL: "needsGalera": "no", "dist_name": os_name, "version_name": OS_INFO[os_info_name]["version_name"], - "arch": build_arch, + "arch": platform, "test_mode": "columnstore", "test_type": "minor", "BB_CI": True, diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index 2752440b5..e939b2bfd 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -143,6 +143,68 @@ rpm_repo_dir() { set -u } +_rpm_get_base_mirror_url() { + local branch=$1 + echo "https://rpm.mariadb.org/$branch" +} + +_rpm_get_base_archive_url() { + local branch=$1 + echo "https://archive.mariadb.org/mariadb-$branch/yum" +} + +_rpm_get_repo_path() { + # Construct a distro repo URL path based on the file hierarchy set up + # by buildbot / release scripts. + local dist_name=$1 + local version_number=$2 + local arch=$3 + local path="${dist_name}${version_number}-${arch}" + + # Special handling for centos 9 + # This uses the previous hierarchy in release scripts instead of a flattened + # path. + if [[ "$dist_name" == "centos" && "$version_number" -ge 9 ]]; then + # Check if architecture is amd64 and convert to x86_64 + if [[ "$arch" == "amd64" ]]; then + arch="x86_64" + fi + path="$dist_name/$version_number/$arch" + fi + + echo "$path" +} + +rpm_get_mirror_url() { + # Return full URL to corresponding distro repo on the mariadb mirror. + local branch=$1 + local dist_name=$2 + local version_number=$3 + local arch=$4 + local base + local path + + base=$(_rpm_get_base_mirror_url "$branch") + path=$(_rpm_get_repo_path "$dist_name" "$version_number" "$arch") + # Print the final constructed URL + echo "${base}/${path}" +} + +rpm_get_archive_url() { + # Return full URL to corresponding distro repo on the archive. + local branch=$1 + local dist_name=$2 + local version_number=$3 + local arch=$4 + local base + local path + + base=$(_rpm_get_base_archive_url "$branch") + path=$(_rpm_get_repo_path "$dist_name" "$version_number" "$arch") + # Print the final constructed URL + echo "${base}/${path}" +} + rpm_pkg() { # ID_LIKE may not exist set +u @@ -279,15 +341,25 @@ rpm_setup_mariadb_mirror() { bb_log_err "missing the branch variable" exit 1 } - branch=$1 + [[ -n $2 ]] || { + bb_log_err "missing the mirror_url variable" + exit 1 + } + [[ -n $3 ]] || { + bb_log_err "missing the archive_url variable" + exit 1 + } + local branch=$1 + local mirror_url=$2 + local archive_url=$3 + bb_log_info "setup MariaDB repository for $branch branch" command -v wget >/dev/null || { bb_log_err "wget command not found" exit 1 } - #//TEMP it's probably better to install the last stable release here...? - mirror_url="https://rpm.mariadb.org/$branch/$arch" - archive_url="https://archive.mariadb.org/mariadb-$branch/yum/$arch" + + local baseurl if wget -q --spider "$mirror_url"; then baseurl="$mirror_url" elif wget -q --spider "$archive_url"; then @@ -297,7 +369,7 @@ rpm_setup_mariadb_mirror() { # since we know it will always fail. But apparently, it's not going to # happen soon in BB. Once done though, replace the warning with an error # and use a non-zero exit code. - bb_log_warn "rpm_setup_mariadb_mirror: $branch packages for $dist_name $version_name does not exist on https://rpm.mariadb.org/" + bb_log_warn "rpm_setup_mariadb_mirror: $branch packages do not exist on either $mirror_url or $archive_url" exit 0 fi cat < dict[str, str]: +def envFromProperties(properties: list[str]) -> dict[str, str]: d = {} - for e in envlist: - d[e] = util.Interpolate(f"%(prop:{e})s") - d["tarbuildnum"] = util.Interpolate("%(prop:tarbuildnum)s") - d["development_branch"] = DEVELOPMENT_BRANCH + for prop_name in properties: + d[prop_name] = util.Interpolate(f"%(prop:{prop_name})s") return d