Skip to content

Retry uv pip install on cargo-related transient failures #64588

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
potiuk wants to merge 1 commit into from
Closed

Retry uv pip install on cargo-related transient failures #64588

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 40 additions & 18 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1195,6 +1195,40 @@ COPY <<"EOF" /install_airflow_when_building_images.sh

. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

function run_uv_with_cargo_retry() {
local max_attempts=5
local sleep_seconds=30
local attempt
local exit_code
local output_file
output_file=$(mktemp)
# shellcheck disable=SC2064
trap "rm -f ${output_file}" RETURN
for attempt in $(seq 1 ${max_attempts}); do
echo "${COLOR_BLUE}Attempt ${attempt} of ${max_attempts} for: $*${COLOR_RESET}"
set +e
set -x
"$@" 2>&1 | tee "${output_file}"
exit_code=${PIPESTATUS[0]}
set +x
set -e
if [[ ${exit_code} == 0 ]]; then
return 0
fi
if ! grep -qi -e "cargo" -e "maturin" "${output_file}"; then
echo "${COLOR_YELLOW}Failure is not cargo-related, not retrying.${COLOR_RESET}"
return ${exit_code}
fi
if [[ ${attempt} -lt ${max_attempts} ]]; then
echo "${COLOR_YELLOW}Attempt ${attempt} failed with cargo-related error. Sleeping ${sleep_seconds}s before retry...${COLOR_RESET}"
sleep ${sleep_seconds}
else
echo "${COLOR_RED}All ${max_attempts} attempts failed.${COLOR_RESET}"
fi
done
return ${exit_code}
}

function install_from_sources() {
local extra_sync_flags
extra_sync_flags=""
Expand All @@ -1216,24 +1250,20 @@ function install_from_sources() {
# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
set -x
uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen \
run_uv_with_cargo_retry uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen \
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python
else
set +x
echo
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
echo
# Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving.
# --no-binary-package is needed in order to avoid libxml and xmlsec using different version of
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
set -x
if ! uv sync --all-packages --frozen --group dev --group docs --group docs-gen \
if ! run_uv_with_cargo_retry uv sync --all-packages --frozen --group dev --group docs --group docs-gen \
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python; then
set +x
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
echo
echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}"
Expand All @@ -1245,11 +1275,9 @@ function install_from_sources() {
echo
echo "${COLOR_BLUE}Falling back to re-resolving dependencies (uv sync without --frozen).${COLOR_RESET}"
echo
set -x
uv sync --all-packages --group dev --group docs --group docs-gen \
run_uv_with_cargo_retry uv sync --all-packages --group dev --group docs --group docs-gen \
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python
set +x
fi
fi
}
Expand All @@ -1276,16 +1304,12 @@ function install_from_external_spec() {
echo
echo "${COLOR_BLUE}Installing all packages with highest resolutions. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
echo
set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
set +x
run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
else
echo
echo "${COLOR_BLUE}Installing all packages with constraints. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
echo
set -x
if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
set +x
if ! run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
echo
echo "${COLOR_RED}Failing because constraints installation failed and fallback is disabled.${COLOR_RESET}"
Expand All @@ -1297,9 +1321,7 @@ function install_from_external_spec() {
echo
echo "${COLOR_BLUE}Falling back to no-constraints installation.${COLOR_RESET}"
echo
set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
set +x
run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
fi
fi
}
Expand Down
58 changes: 40 additions & 18 deletions Dockerfile.ci
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -900,6 +900,40 @@ COPY <<"EOF" /install_airflow_when_building_images.sh

. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

function run_uv_with_cargo_retry() {
local max_attempts=5
local sleep_seconds=30
local attempt
local exit_code
local output_file
output_file=$(mktemp)
# shellcheck disable=SC2064
trap "rm -f ${output_file}" RETURN
for attempt in $(seq 1 ${max_attempts}); do
echo "${COLOR_BLUE}Attempt ${attempt} of ${max_attempts} for: $*${COLOR_RESET}"
set +e
set -x
"$@" 2>&1 | tee "${output_file}"
exit_code=${PIPESTATUS[0]}
set +x
set -e
if [[ ${exit_code} == 0 ]]; then
return 0
fi
if ! grep -qi -e "cargo" -e "maturin" "${output_file}"; then
echo "${COLOR_YELLOW}Failure is not cargo-related, not retrying.${COLOR_RESET}"
return ${exit_code}
fi
if [[ ${attempt} -lt ${max_attempts} ]]; then
echo "${COLOR_YELLOW}Attempt ${attempt} failed with cargo-related error. Sleeping ${sleep_seconds}s before retry...${COLOR_RESET}"
sleep ${sleep_seconds}
else
echo "${COLOR_RED}All ${max_attempts} attempts failed.${COLOR_RESET}"
fi
done
return ${exit_code}
}

function install_from_sources() {
local extra_sync_flags
extra_sync_flags=""
Expand All @@ -921,24 +955,20 @@ function install_from_sources() {
# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
set -x
uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen \
run_uv_with_cargo_retry uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen \
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python
else
set +x
echo
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
echo
# Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving.
# --no-binary-package is needed in order to avoid libxml and xmlsec using different version of
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
set -x
if ! uv sync --all-packages --frozen --group dev --group docs --group docs-gen \
if ! run_uv_with_cargo_retry uv sync --all-packages --frozen --group dev --group docs --group docs-gen \
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python; then
set +x
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
echo
echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}"
Expand All @@ -950,11 +980,9 @@ function install_from_sources() {
echo
echo "${COLOR_BLUE}Falling back to re-resolving dependencies (uv sync without --frozen).${COLOR_RESET}"
echo
set -x
uv sync --all-packages --group dev --group docs --group docs-gen \
run_uv_with_cargo_retry uv sync --all-packages --group dev --group docs --group docs-gen \
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python
set +x
fi
fi
}
Expand All @@ -981,16 +1009,12 @@ function install_from_external_spec() {
echo
echo "${COLOR_BLUE}Installing all packages with highest resolutions. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
echo
set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
set +x
run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
else
echo
echo "${COLOR_BLUE}Installing all packages with constraints. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
echo
set -x
if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
set +x
if ! run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
echo
echo "${COLOR_RED}Failing because constraints installation failed and fallback is disabled.${COLOR_RESET}"
Expand All @@ -1002,9 +1026,7 @@ function install_from_external_spec() {
echo
echo "${COLOR_BLUE}Falling back to no-constraints installation.${COLOR_RESET}"
echo
set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
set +x
run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
fi
fi
}
Expand Down
63 changes: 45 additions & 18 deletions scripts/docker/install_airflow_when_building_images.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,45 @@
# shellcheck source=scripts/docker/common.sh
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

# Retry wrapper for uv commands that may fail with transient cargo/maturin build errors.
# Workaround for https://github.com/astral-sh/uv/issues/18801
# Usage: run_uv_with_cargo_retry <command...>
# Returns the exit code of the command (non-zero only after all retries exhausted for cargo errors,
# or immediately for non-cargo failures).
function run_uv_with_cargo_retry() {
local max_attempts=5
local sleep_seconds=30
local attempt
local exit_code
local output_file
output_file=$(mktemp)
# shellcheck disable=SC2064
trap "rm -f ${output_file}" RETURN
for attempt in $(seq 1 ${max_attempts}); do
echo "${COLOR_BLUE}Attempt ${attempt} of ${max_attempts} for: $*${COLOR_RESET}"
set +e
set -x
"$@" 2>&1 | tee "${output_file}"
exit_code=${PIPESTATUS[0]}
set +x
set -e
if [[ ${exit_code} == 0 ]]; then
return 0
fi
if ! grep -qi -e "cargo" -e "maturin" "${output_file}"; then
echo "${COLOR_YELLOW}Failure is not cargo-related, not retrying.${COLOR_RESET}"
return ${exit_code}
fi
if [[ ${attempt} -lt ${max_attempts} ]]; then
echo "${COLOR_YELLOW}Attempt ${attempt} failed with cargo-related error. Sleeping ${sleep_seconds}s before retry...${COLOR_RESET}"
sleep ${sleep_seconds}
else
echo "${COLOR_RED}All ${max_attempts} attempts failed.${COLOR_RESET}"
fi
done
return ${exit_code}
}

function install_from_sources() {
local extra_sync_flags
extra_sync_flags=""
Expand All @@ -56,24 +95,20 @@ function install_from_sources() {
# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
set -x
uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen \
run_uv_with_cargo_retry uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen \
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python
else
set +x
echo
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
echo
# Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving.
# --no-binary-package is needed in order to avoid libxml and xmlsec using different version of
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
set -x
if ! uv sync --all-packages --frozen --group dev --group docs --group docs-gen \
if ! run_uv_with_cargo_retry uv sync --all-packages --frozen --group dev --group docs --group docs-gen \
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python; then
set +x
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
echo
echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}"
Expand All @@ -85,11 +120,9 @@ function install_from_sources() {
echo
echo "${COLOR_BLUE}Falling back to re-resolving dependencies (uv sync without --frozen).${COLOR_RESET}"
echo
set -x
uv sync --all-packages --group dev --group docs --group docs-gen \
run_uv_with_cargo_retry uv sync --all-packages --group dev --group docs --group docs-gen \
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python
set +x
fi
fi
}
Expand All @@ -116,16 +149,12 @@ function install_from_external_spec() {
echo
echo "${COLOR_BLUE}Installing all packages with highest resolutions. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
echo
set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
set +x
run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
else
echo
echo "${COLOR_BLUE}Installing all packages with constraints. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
echo
set -x
if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
set +x
if ! run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
echo
echo "${COLOR_RED}Failing because constraints installation failed and fallback is disabled.${COLOR_RESET}"
Expand All @@ -137,9 +166,7 @@ function install_from_external_spec() {
echo
echo "${COLOR_BLUE}Falling back to no-constraints installation.${COLOR_RESET}"
echo
set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
set +x
run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
fi
fi
}
Expand Down
Loading
Loading