Skip to content

Commit 29a4456

Browse files
keighrimkeighrim
committed
updated build all containers workflow, added script for future generation
1 parent a69bace commit 29a4456

File tree

2 files changed

+182
-25
lines changed

2 files changed

+182
-25
lines changed

‎.github/workflows/containers.yml‎

Lines changed: 80 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1+
# This file is auto-generated. Do not edit manually.
2+
13
name: "📦 Publish (base images)"
24

35
on:
46
push:
5-
tags:
7+
tags:
68
- '[0-9]+.[0-9]+.[0-9]+'
79
workflow_dispatch:
810
inputs:
@@ -19,11 +21,11 @@ jobs:
1921
steps:
2022
- name: "📌 Set VERSION value from dispatch inputs"
2123
id: get_version_dispatch
22-
if: ${{ github.event_name == 'workflow_dispatch' }}
24+
if: ${{ github.event_name == 'workflow_dispatch' }}
2325
run: echo "VERSION=${{ github.event.inputs.version }}" >> $GITHUB_ENV
2426
- name: "📌 Set VERSION value from pushed tag"
2527
id: get_version_tag
26-
if: ${{ github.event_name == 'push' }}
28+
if: ${{ github.event_name == 'push' }}
2729
run: echo "VERSION=$(echo "${{ github.ref }}" | cut -d/ -f3)" >> $GITHUB_ENV
2830
- name: "🏷 Record VERSION for follow-up jobs"
2931
id: output_version
@@ -51,6 +53,24 @@ jobs:
5153
buildfilename: './container/Containerfile'
5254
version: ${{ needs.set-version.outputs.version }}
5355

56+
call-build-ffmpeg:
57+
name: "🤙 Call container workflow with `ffmpeg`"
58+
needs: ['set-version', 'call-build-base']
59+
uses: ./.github/workflows/container.yml
60+
secrets: inherit
61+
with:
62+
buildfilename: './container/ffmpeg.containerfile'
63+
version: ${{ needs.set-version.outputs.version }}
64+
65+
call-build-hf:
66+
name: "🤙 Call container workflow with `hf`"
67+
needs: ['set-version', 'call-build-base']
68+
uses: ./.github/workflows/container.yml
69+
secrets: inherit
70+
with:
71+
buildfilename: './container/hf.containerfile'
72+
version: ${{ needs.set-version.outputs.version }}
73+
5474
call-build-jdk8:
5575
name: "🤙 Call container workflow with `jdk8`"
5676
needs: ['set-version', 'call-build-base']
@@ -69,31 +89,31 @@ jobs:
6989
buildfilename: './container/tf2.containerfile'
7090
version: ${{ needs.set-version.outputs.version }}
7191

72-
call-build-torch:
73-
name: "🤙 Call container workflow with `torch`"
92+
call-build-torch2:
93+
name: "🤙 Call container workflow with `torch2`"
7494
needs: ['set-version', 'call-build-base']
7595
uses: ./.github/workflows/container.yml
7696
secrets: inherit
7797
with:
78-
buildfilename: './container/torch.containerfile'
98+
buildfilename: './container/torch2.containerfile'
7999
version: ${{ needs.set-version.outputs.version }}
80100

81-
call-build-torch2:
82-
name: "🤙 Call container workflow with `torch2`"
101+
call-build-transformers4:
102+
name: "🤙 Call container workflow with `transformers4`"
83103
needs: ['set-version', 'call-build-base']
84104
uses: ./.github/workflows/container.yml
85105
secrets: inherit
86106
with:
87-
buildfilename: './container/torch2.containerfile'
107+
buildfilename: './container/transformers4.containerfile'
88108
version: ${{ needs.set-version.outputs.version }}
89109

90-
call-build-ffmpeg:
91-
name: "🤙 Call container workflow with `ffmpeg`"
92-
needs: ['set-version', 'call-build-base']
110+
call-build-ffmpeg-hf:
111+
name: "🤙 Call container workflow with `ffmpeg-hf`"
112+
needs: ['set-version', 'call-build-ffmpeg']
93113
uses: ./.github/workflows/container.yml
94114
secrets: inherit
95115
with:
96-
buildfilename: './container/ffmpeg.containerfile'
116+
buildfilename: './container/ffmpeg-hf.containerfile'
97117
version: ${{ needs.set-version.outputs.version }}
98118

99119
call-build-ffmpeg-tf2:
@@ -105,22 +125,22 @@ jobs:
105125
buildfilename: './container/ffmpeg-tf2.containerfile'
106126
version: ${{ needs.set-version.outputs.version }}
107127

108-
call-build-ffmpeg-torch:
109-
name: "🤙 Call container workflow with `ffmpeg-torch`"
128+
call-build-ffmpeg-torch2:
129+
name: "🤙 Call container workflow with `ffmpeg-torch2`"
110130
needs: ['set-version', 'call-build-ffmpeg']
111131
uses: ./.github/workflows/container.yml
112132
secrets: inherit
113133
with:
114-
buildfilename: './container/ffmpeg-torch.containerfile'
134+
buildfilename: './container/ffmpeg-torch2.containerfile'
115135
version: ${{ needs.set-version.outputs.version }}
116136

117-
call-build-ffmpeg-torch2:
118-
name: "🤙 Call container workflow with `ffmpeg-torch2`"
137+
call-build-ffmpeg-transformers4:
138+
name: "🤙 Call container workflow with `ffmpeg-transformers4`"
119139
needs: ['set-version', 'call-build-ffmpeg']
120140
uses: ./.github/workflows/container.yml
121141
secrets: inherit
122142
with:
123-
buildfilename: './container/ffmpeg-torch2.containerfile'
143+
buildfilename: './container/ffmpeg-transformers4.containerfile'
124144
version: ${{ needs.set-version.outputs.version }}
125145

126146
call-build-opencv4:
@@ -132,22 +152,22 @@ jobs:
132152
buildfilename: './container/opencv4.containerfile'
133153
version: ${{ needs.set-version.outputs.version }}
134154

135-
call-build-opencv4-tf2:
136-
name: "🤙 Call container workflow with `opencv4-tf2`"
155+
call-build-opencv4-hf:
156+
name: "🤙 Call container workflow with `opencv4-hf`"
137157
needs: ['set-version', 'call-build-opencv4']
138158
uses: ./.github/workflows/container.yml
139159
secrets: inherit
140160
with:
141-
buildfilename: './container/opencv4-tf2.containerfile'
161+
buildfilename: './container/opencv4-hf.containerfile'
142162
version: ${{ needs.set-version.outputs.version }}
143163

144-
call-build-opencv4-torch:
145-
name: "🤙 Call container workflow with `opencv4-torch`"
164+
call-build-opencv4-tf2:
165+
name: "🤙 Call container workflow with `opencv4-tf2`"
146166
needs: ['set-version', 'call-build-opencv4']
147167
uses: ./.github/workflows/container.yml
148168
secrets: inherit
149169
with:
150-
buildfilename: './container/opencv4-torch.containerfile'
170+
buildfilename: './container/opencv4-tf2.containerfile'
151171
version: ${{ needs.set-version.outputs.version }}
152172

153173
call-build-opencv4-torch2:
@@ -159,3 +179,38 @@ jobs:
159179
buildfilename: './container/opencv4-torch2.containerfile'
160180
version: ${{ needs.set-version.outputs.version }}
161181

182+
call-build-opencv4-transformers4:
183+
name: "🤙 Call container workflow with `opencv4-transformers4`"
184+
needs: ['set-version', 'call-build-opencv4']
185+
uses: ./.github/workflows/container.yml
186+
secrets: inherit
187+
with:
188+
buildfilename: './container/opencv4-transformers4.containerfile'
189+
version: ${{ needs.set-version.outputs.version }}
190+
191+
call-build-tf2-hf:
192+
name: "🤙 Call container workflow with `tf2-hf`"
193+
needs: ['set-version', 'call-build-tf2']
194+
uses: ./.github/workflows/container.yml
195+
secrets: inherit
196+
with:
197+
buildfilename: './container/tf2-hf.containerfile'
198+
version: ${{ needs.set-version.outputs.version }}
199+
200+
call-build-ffmpeg-tf2-hf:
201+
name: "🤙 Call container workflow with `ffmpeg-tf2-hf`"
202+
needs: ['set-version', 'call-build-ffmpeg-tf2']
203+
uses: ./.github/workflows/container.yml
204+
secrets: inherit
205+
with:
206+
buildfilename: './container/ffmpeg-tf2-hf.containerfile'
207+
version: ${{ needs.set-version.outputs.version }}
208+
209+
call-build-opencv4-tf2-hf:
210+
name: "🤙 Call container workflow with `opencv4-tf2-hf`"
211+
needs: ['set-version', 'call-build-opencv4-tf2']
212+
uses: ./.github/workflows/container.yml
213+
secrets: inherit
214+
with:
215+
buildfilename: './container/opencv4-tf2-hf.containerfile'
216+
version: ${{ needs.set-version.outputs.version }}

‎container/generate_containers_yml.py‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
from pathlib import Path
2+
3+
def scan_container_files(directory):
4+
"""Scan the container directory for Containerfile and *.containerfile files."""
5+
container_files = [f.name for f in Path(directory).iterdir() if f.is_file() and (f.name.endswith('.containerfile') or f.name == 'Containerfile')]
6+
return container_files
7+
8+
def determine_dependencies(container_files):
9+
"""Determine dependencies based on file names."""
10+
dependencies = {}
11+
for file in container_files:
12+
if '-' in file:
13+
base = file.rsplit('-', 1)[0]
14+
dependencies[file] = base.split('-')
15+
else:
16+
# If the file is a base image, it has no dependencies
17+
if file == 'opencv4.containerfile':
18+
# Special case for opencv4.containerfile which depends on ffmpeg
19+
# This is a dependency specified before the filename convention
20+
dependencies[file] = ['ffmpeg']
21+
elif file != 'Containerfile':
22+
dependencies[file] = []
23+
print(dependencies) # Debugging output to check dependencies
24+
return dependencies
25+
26+
def generate_workflow(container_files, dependencies, output_file):
27+
"""Generate the containers.yml workflow file."""
28+
with open(output_file, 'w') as f:
29+
# Add a comment indicating the file is auto-generated
30+
f.write("# This file is auto-generated. Do not edit manually.\n\n")
31+
32+
# Write the static configuration part
33+
f.write("name: \"📦 Publish (base images)\"\n\n")
34+
f.write("on:\n push:\n tags:\n - '[0-9]+.[0-9]+.[0-9]+'\n workflow_dispatch:\n inputs:\n version:\n description: 'version to tag images'\n required: true\n\n")
35+
f.write("jobs:\n")
36+
f.write(" set-version:\n")
37+
f.write(" runs-on: ubuntu-latest\n")
38+
f.write(" name: \"🏷 Set version value\"\n")
39+
f.write(" outputs:\n")
40+
f.write(" version: ${{ steps.output_version.outputs.version }}\n")
41+
f.write(" steps:\n")
42+
f.write(" - name: \"📌 Set VERSION value from dispatch inputs\"\n")
43+
f.write(" id: get_version_dispatch\n")
44+
f.write(" if: ${{ github.event_name == 'workflow_dispatch' }}\n")
45+
f.write(" run: echo \"VERSION=${{ github.event.inputs.version }}\" >> $GITHUB_ENV\n")
46+
f.write(" - name: \"📌 Set VERSION value from pushed tag\"\n")
47+
f.write(" id: get_version_tag\n")
48+
f.write(" if: ${{ github.event_name == 'push' }}\n")
49+
f.write(" run: echo \"VERSION=$(echo \"${{ github.ref }}\" | cut -d/ -f3)\" >> $GITHUB_ENV\n")
50+
f.write(" - name: \"🏷 Record VERSION for follow-up jobs\"\n")
51+
f.write(" id: output_version\n")
52+
f.write(" run: |\n")
53+
f.write(" echo \"version=${{ env.VERSION }}\" >> $GITHUB_OUTPUT\n")
54+
f.write(" check-deployment:\n")
55+
f.write(" name: \"✅ PyPI deployment check\"\n")
56+
f.write(" runs-on: ubuntu-latest\n")
57+
f.write(" needs: ['set-version']\n")
58+
f.write(" steps:\n")
59+
f.write(" - name: \"⏱️ Wait up to 20 minutes for the new clams-python is deployed on PyPI\"\n")
60+
f.write(" uses: nev7n/wait_for_response@v1\n")
61+
f.write(" with:\n")
62+
f.write(" url: \"https://pypi.org/project/clams-python/${{ needs.set-version.outputs.version }}/\"\n")
63+
f.write(" responseCode: 200\n")
64+
f.write(" timeout: 1200000\n")
65+
f.write(" interval: 5000\n")
66+
67+
# Sort jobs: base first, then by dependency length and filename
68+
sorted_files = sorted(container_files, key=lambda x: (x != 'Containerfile', len(dependencies.get(x, [])), '-'.join(dependencies.get(x, [])), x))
69+
70+
for file in sorted_files:
71+
job_name = 'base' if file == 'Containerfile' else file.replace('.containerfile', '').replace('.', '-').replace('_', '-')
72+
f.write(f"\n call-build-{job_name}:\n")
73+
human_friendly_job_name = "the base" if job_name == 'base' else f"`{job_name}`"
74+
f.write(f" name: \"🤙 Call container workflow with {human_friendly_job_name}\"\n")
75+
dependency_job = dependencies.get(file, None)
76+
if dependency_job is not None:
77+
if len(dependency_job) > 0:
78+
f.write(f" needs: ['set-version', 'call-build-{'-'.join(dependency_job)}']\n")
79+
else:
80+
f.write(f" needs: ['set-version', 'call-build-base']\n")
81+
else:
82+
f.write(f" needs: ['set-version', 'check-deployment']\n")
83+
f.write(" uses: ./.github/workflows/container.yml\n")
84+
f.write(" secrets: inherit\n")
85+
f.write(" with:\n")
86+
f.write(f" buildfilename: './container/{file}'\n")
87+
f.write(f" version: ${{{{ needs.set-version.outputs.version }}}}\n")
88+
89+
if __name__ == "__main__":
90+
container_dir = Path(__file__).parent
91+
output_file = Path(__file__).parent.parent / ".github/workflows/containers.yml"
92+
93+
print("Scanning container directory...")
94+
container_files = scan_container_files(container_dir)
95+
96+
print("Determining dependencies...")
97+
dependencies = determine_dependencies(container_files)
98+
99+
print("Generating workflow file...")
100+
generate_workflow(container_files, dependencies, output_file)
101+
102+
print(f"Workflow file generated at {output_file}")

0 commit comments

Comments
 (0)