Skip to content

Commit fdc19f3

Browse files
authored
Merge pull request #3 from bryopsida/implement-fetch-images-job
Build image list of running images in cluster
2 parents 9a2cc8e + 0e504ce commit fdc19f3

23 files changed

+628
-92
lines changed

Makefile

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
HELM_NAMESPACE ?= patchwork
2+
HELM_RELEASE_NAME ?= patchwork
3+
HELM_IMAGE_TAG ?= main
4+
5+
helmDeploy:
6+
helm upgrade --install --namespace=$(HELM_NAMESPACE) $(HELM_RELEASE_NAME) charts/patchwork --set image.tag=$(HELM_IMAGE_TAG) --debug --wait $(HELM_EXTRA_ARGS)

README.md

Lines changed: 12 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,18 @@
1-
# Nest.JS Starter Template
1+
# Patchwork
22

3-
## What's Included
3+
## What does this do?
44

5-
- OpenAPI/Swagger
6-
- Health EP `/health`
7-
- Helmet
8-
- Dockerfile
9-
- Helm Chart
10-
- GitHub Action workflows to lint, test, build image, and verify helm chart
5+
This inspects your statefulsets, daemonsets, deployments and checks the associated registries to see if the tag has been updated. If an update is available it triggers a rollout on that resource. This is very early in development and should not be run in production servers.
116

12-
## NPM Scripts
7+
## What are the current limitations
138

14-
| Syntax | Description |
15-
| ------------------ | ------------------------------------------------------------------- |
16-
| minikube:start | Creates a minikube cluster for testing |
17-
| minikube:stop | Stops the minikube cluster |
18-
| minikube:delete | Deletes the minikube cluster |
19-
| minikube:copyImage | Load the build image from build:image into the minikube cluster |
20-
| build | Compile the TypeScript to JavaScript |
21-
| build:image | Build the container image |
22-
| build:docs | Builds the docs folder |
23-
| start:dev | Run the server with hot reloading on save |
24-
| start | Start the server |
25-
| lint | Check if code passes linting rules |
26-
| lint:fix | Automatically fix any formatting or linting rules that can be fixed |
27-
| helm:deploy | Deploy the chart with the local to your cluster |
28-
| helm:uninstall | Delete the chart release from your cluster |
29-
| helm:test | Run the packaged tests (postman) for the helm release |
30-
| test | Run the unit tests |
9+
This is still very early in development and has a few limitations.
3110

32-
## How to use published chart
11+
1. Does not support private registries
12+
2. Does not pull updates for things without `imagePullPolicy = Always`
13+
3. Only supports querying `linux/arm64` image repositories at the moment
14+
4. Only looks for updates to the same tag, IE for cases where base patches have been pushed to a tag
3315

34-
First add the repo `helm repo add nestjs-starter https://bryopsida.github.io/nestjs-starter/`, then fetch updates `helm repo update`, and finally, install with `helm upgrade --install starter nestjs-starter/nestjs-starter --wait`.
16+
## How to deploy
17+
18+
First add the repo `helm repo add patchwork https://bryopsida.github.io/patchwork/`, then fetch updates `helm repo update`, and finally, install with `helm upgrade --install patchwork patchwork/patchwork --wait`.

charts/patchwork/Chart.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ apiVersion: v2
22
name: patchwork
33
description: A Helm chart for Kubernetes
44
type: application
5-
version: 0.2.1
5+
version: 0.2.2
66
appVersion: '0.1.0'
77
dependencies:
88
- name: redis
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{{- if .Values.rbac.create }}
2+
---
3+
apiVersion: rbac.authorization.k8s.io/v1
4+
kind: ClusterRole
5+
metadata:
6+
name: {{ .Release.Name }}-patchwork-updater
7+
rules:
8+
# need to be able to fetch pull secrets to interact with private registries
9+
- apiGroups: [""]
10+
resources: ["secrets"]
11+
verbs: ["get"]
12+
# meed to to be able to fetch indivudal pods to lookup image hashes
13+
# need to be able to lookup images on node and the node arch
14+
- apiGroups: [""]
15+
resources: ["pods", "nodes"]
16+
verbs: ["get", "list", "watch"]
17+
# need to be able to lookup/list and patch apps
18+
- apiGroups: ["apps"]
19+
resources: ["deployments", "daemonsets", "statefulsets"]
20+
verbs: ["get", "list", "watch", "patch"]
21+
---
22+
apiVersion: rbac.authorization.k8s.io/v1
23+
kind: ClusterRoleBinding
24+
metadata:
25+
name: {{ .Release.Name }}-patchwork-updater
26+
subjects:
27+
- kind: ServiceAccount
28+
name: {{ include "nestjs-starter.serviceAccountName" . }}
29+
namespace: {{ .Release.Namespace }}
30+
roleRef:
31+
kind: ClusterRole
32+
name: {{ .Release.Name }}-patchwork-updater
33+
apiGroup: rbac.authorization.k8s.io
34+
{{- end }}

charts/patchwork/values.yaml

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ imagePullSecrets: []
1313
nameOverride: ''
1414
fullnameOverride: ''
1515

16+
rbac:
17+
# if set, the chart will create a clusterrole and rolebinding to give it the permissions its needs
18+
create: truewq
1619
serviceAccount:
1720
# Specifies whether a service account should be created
1821
create: true
@@ -62,11 +65,11 @@ ingress:
6265

6366
resources:
6467
limits:
65-
cpu: 100m
66-
memory: 128Mi
68+
cpu: 500m
69+
memory: 256Mi
6770
requests:
68-
cpu: 100m
69-
memory: 128Mi
71+
cpu: 250m
72+
memory: 256Mi
7073

7174
autoscaling:
7275
enabled: false

package-lock.json

Lines changed: 60 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@
2020
"services:start": "docker-compose up -d",
2121
"services:stop": "docker-compose down",
2222
"start": "nest start",
23-
"start:dev": "nest start --watch | pino-pretty",
23+
"start:dev:prettyLogs": "nest start --watch | pino-pretty",
24+
"start:dev": "nest start --watch | jq -R '. as $line | try (fromjson) catch $line'",
2425
"test": "jest --coverage",
2526
"lint": "eslint --ext .ts src/ && prettier --check .",
2627
"lint:fix": "eslint --ext .ts src/ --fix && prettier --write .",
@@ -37,6 +38,7 @@
3738
"@nestjs/schedule": "^3.0.1",
3839
"@nestjs/swagger": "^7.0.1",
3940
"@nestjs/terminus": "^10.0.1",
41+
"@snyk/docker-registry-v2-client": "^2.10.0",
4042
"bull": "^4.10.4",
4143
"fast-json-patch": "^3.1.1",
4244
"helmet": "^7.0.0",

src/analyzer/analyzer.module.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import { RegistryService } from './registry.service'
44
import { BullModule } from '@nestjs/bull'
55
import { ImageListWorker } from './image-list.consumer'
66
import { TaskRegisterService } from './task-register.service'
7+
import { KubernetesModule } from '../kubernetes/kubernetes.module'
8+
import { ImageDescriptorWorker } from './image-descriptor.consumer'
79

810
@Module({
911
imports: [
@@ -13,8 +15,13 @@ import { TaskRegisterService } from './task-register.service'
1315
BullModule.registerQueue({
1416
name: 'analyzer.check.updates',
1517
}),
18+
BullModule.registerQueue({
19+
name: 'patcher.update',
20+
}),
21+
KubernetesModule,
1622
],
1723
providers: [
24+
ImageDescriptorWorker,
1825
ImageService,
1926
RegistryService,
2027
ImageListWorker,
Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
import { Processor, Process, InjectQueue } from '@nestjs/bull'
2+
import { Logger } from '@nestjs/common'
3+
import { Job, Queue } from 'bull'
4+
import { ImageDescriptor } from '../kubernetes/k8s.service'
5+
import { getManifest, contentTypes } from '@snyk/docker-registry-v2-client'
6+
@Processor('analyzer.check.updates')
7+
export class ImageDescriptorWorker {
8+
private readonly logger = new Logger(ImageDescriptorWorker.name)
9+
private readonly patchQueue: Queue
10+
11+
constructor(@InjectQueue('patcher.update') queue: Queue) {
12+
this.patchQueue = queue
13+
}
14+
15+
@Process()
16+
async fetchImageList(job: Job<ImageDescriptor>) {
17+
try {
18+
this.logger.debug(
19+
`Checking for updates of ${job.data.repository}:${
20+
job.data.tag
21+
} used by ${
22+
job.data.owner.namespace
23+
}/${job.data.owner.type.toString()}/${job.data.owner.name}`
24+
)
25+
// split on / check for a . in the first snippet
26+
const parts = job.data.repository.split('/')
27+
// this may not be universarlly true
28+
const isDockerIo = parts[0].indexOf('.') === -1
29+
const dockerIoRegistryDomain = 'registry-1.docker.io'
30+
const dockerIo = 'docker.io'
31+
const registry = isDockerIo
32+
? 'registry-1.docker.io'
33+
: parts[0] === dockerIo
34+
? dockerIoRegistryDomain
35+
: parts[0]
36+
const repo = isDockerIo
37+
? parts.length === 1
38+
? `library/${job.data.repository}`
39+
: job.data.repository
40+
: parts[0] === dockerIo
41+
? job.data.repository.substring(dockerIo.length + 1)
42+
: job.data.repository.substring(registry.length + 1)
43+
this.logger.debug(
44+
`Fetching manifest for registry = ${registry}, repo = ${repo}, tag = ${job.data.tag}`
45+
)
46+
const reqOptions =
47+
registry === 'ghcr.io'
48+
? {
49+
acceptManifest: `${contentTypes.MANIFEST_V2}, ${contentTypes.MANIFEST_LIST_V2}, ${contentTypes.OCI_INDEX_V1}, ${contentTypes.OCI_MANIFEST_V1}`,
50+
}
51+
: undefined
52+
const manifest = await getManifest(
53+
registry,
54+
repo,
55+
job.data.tag,
56+
undefined,
57+
undefined,
58+
reqOptions,
59+
{
60+
os: 'linux',
61+
architecture: job.data.arch,
62+
}
63+
)
64+
if (manifest == null || manifest?.indexDigest == null) {
65+
this.logger.warn(
66+
'Failed to get a workable manifest for %s with tag %s from registry %s',
67+
repo,
68+
job.data.tag,
69+
registry
70+
)
71+
}
72+
this.logger.debug(
73+
`Fetched manifest digest = ${manifest?.indexDigest}, running hash = ${job.data.hash}, repo = ${job.data.repository}`,
74+
manifest
75+
)
76+
if (
77+
manifest.indexDigest !== job.data.hash &&
78+
manifest.indexDigest != null
79+
) {
80+
this.logger.warn(
81+
`Found an update for ${registry}/${repo}:${job.data.tag}`
82+
)
83+
// queueu work for the patcher
84+
await this.patchQueue.add({
85+
...job.data,
86+
...{
87+
currentSha: job.data.hash,
88+
targetSha: manifest.indexDigest,
89+
},
90+
})
91+
return {
92+
detectedUpdate: true,
93+
current: job.data.hash,
94+
detectedLatest: manifest.indexDigest,
95+
}
96+
} else {
97+
return {
98+
detectedUpdate: false,
99+
current: job.data.hash,
100+
detectedLatest: manifest.indexDigest,
101+
}
102+
}
103+
} catch (err) {
104+
this.logger.error(
105+
`Error while checking for updates for ${job.data.repository}:${job.data.tag}: ${err.message}`,
106+
null,
107+
err
108+
)
109+
throw err
110+
}
111+
}
112+
}

0 commit comments

Comments
 (0)