Skip to content

Commit 2a2d158

Browse files
scooby87scooby87
committed
fix(monitoring): clean up orphaned storage PVCs and TLS secrets on tenant module delete (#3091)
When the monitoring tenant module is disabled, the vm-operator storage PVCs (VictoriaMetrics/VictoriaLogs) and the alerta/grafana TLS secrets were left orphaned. Add a post-delete cleanup hook that deletes the PVCs by the managed-by=vm-operator label and the two TLS secrets by name, plus a test target and helm-unittest suite (the package had neither). CNPG PVCs are not targeted (CNPG cleans them). The cleanup Job is hardened (non-root, read-only root fs, dropped caps, resource limits); failures are logged, never blocking teardown. NOTE: disabling monitoring now permanently removes its storage PVCs and the data they hold. Signed-off-by: Алексей Артамонов <[email protected]>
1 parent 374cac1 commit 2a2d158

3 files changed

Lines changed: 242 additions & 0 deletions

File tree

‎packages/extra/monitoring/Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@ NAME=monitoring
55
include ../../../hack/common-envs.mk
66
include ../../../hack/package.mk
77

8+
.PHONY: test
9+
test:
10+
helm unittest .
11+
812
generate:
913
cozyvalues-gen -v values.yaml -s values.schema.json -r README.md
1014
../../../hack/update-crd.sh

‎packages/extra/monitoring/templates/hooks/cleanup.yaml‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
---
2+
apiVersion: batch/v1
3+
kind: Job
4+
metadata:
5+
name: {{ .Release.Name }}-cleanup
6+
labels:
7+
app.kubernetes.io/instance: {{ .Release.Name }}
8+
annotations:
9+
"helm.sh/hook": post-delete
10+
"helm.sh/hook-weight": "10"
11+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
12+
spec:
13+
template:
14+
metadata:
15+
labels:
16+
app.kubernetes.io/instance: {{ .Release.Name }}
17+
policy.cozystack.io/allow-to-apiserver: "true"
18+
spec:
19+
serviceAccountName: {{ .Release.Name }}-cleanup
20+
restartPolicy: Never
21+
securityContext:
22+
runAsNonRoot: true
23+
runAsUser: 65534
24+
runAsGroup: 65534
25+
seccompProfile:
26+
type: RuntimeDefault
27+
containers:
28+
- name: cleanup
29+
image: docker.io/clastix/kubectl:v1.32
30+
securityContext:
31+
allowPrivilegeEscalation: false
32+
readOnlyRootFilesystem: true
33+
capabilities:
34+
drop:
35+
- ALL
36+
env:
37+
- name: HOME
38+
value: /tmp
39+
resources:
40+
requests:
41+
cpu: 10m
42+
memory: 64Mi
43+
limits:
44+
cpu: 200m
45+
memory: 128Mi
46+
command:
47+
- /bin/sh
48+
- -c
49+
- |
50+
kubectl delete pvc -n {{ .Release.Namespace }} -l managed-by=vm-operator --ignore-not-found \
51+
|| echo "WARNING: VM storage PVC cleanup failed for {{ .Release.Name }}; orphaned PVCs may remain" >&2
52+
kubectl delete secret -n {{ .Release.Namespace }} alerta-tls grafana-ingress-tls --ignore-not-found \
53+
|| echo "WARNING: TLS secret cleanup failed for {{ .Release.Name }}; orphaned secrets may remain" >&2
54+
volumeMounts:
55+
- name: tmp
56+
mountPath: /tmp
57+
volumes:
58+
- name: tmp
59+
emptyDir: {}
60+
---
61+
apiVersion: v1
62+
kind: ServiceAccount
63+
metadata:
64+
name: {{ .Release.Name }}-cleanup
65+
labels:
66+
app.kubernetes.io/instance: {{ .Release.Name }}
67+
annotations:
68+
"helm.sh/hook": post-delete
69+
"helm.sh/hook-weight": "0"
70+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
71+
---
72+
apiVersion: rbac.authorization.k8s.io/v1
73+
kind: Role
74+
metadata:
75+
name: {{ .Release.Name }}-cleanup
76+
labels:
77+
app.kubernetes.io/instance: {{ .Release.Name }}
78+
annotations:
79+
"helm.sh/hook": post-delete
80+
"helm.sh/hook-weight": "5"
81+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
82+
rules:
83+
- apiGroups: [""]
84+
resources: ["persistentvolumeclaims"]
85+
verbs: ["get", "list", "delete"]
86+
- apiGroups: [""]
87+
resources: ["secrets"]
88+
resourceNames: ["alerta-tls", "grafana-ingress-tls"]
89+
verbs: ["get", "delete"]
90+
---
91+
apiVersion: rbac.authorization.k8s.io/v1
92+
kind: RoleBinding
93+
metadata:
94+
name: {{ .Release.Name }}-cleanup
95+
labels:
96+
app.kubernetes.io/instance: {{ .Release.Name }}
97+
annotations:
98+
"helm.sh/hook": post-delete
99+
"helm.sh/hook-weight": "5"
100+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
101+
roleRef:
102+
apiGroup: rbac.authorization.k8s.io
103+
kind: Role
104+
name: {{ .Release.Name }}-cleanup
105+
subjects:
106+
- kind: ServiceAccount
107+
name: {{ .Release.Name }}-cleanup

‎packages/extra/monitoring/tests/cleanup_test.yaml‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
suite: monitoring post-delete cleanup hook removes orphaned VM PVCs and TLS secrets
2+
3+
release:
4+
name: monitoring
5+
namespace: tenant-root
6+
7+
templates:
8+
- templates/hooks/cleanup.yaml
9+
10+
tests:
11+
- it: renders the cleanup Job as a post-delete hook with the hardened pod spec
12+
documentSelector:
13+
path: kind
14+
value: Job
15+
asserts:
16+
- equal:
17+
path: metadata.name
18+
value: monitoring-cleanup
19+
- equal:
20+
path: metadata.annotations["helm.sh/hook"]
21+
value: post-delete
22+
- equal:
23+
path: metadata.annotations["helm.sh/hook-weight"]
24+
value: "10"
25+
- equal:
26+
path: metadata.annotations["helm.sh/hook-delete-policy"]
27+
value: before-hook-creation,hook-succeeded
28+
- equal:
29+
path: spec.template.metadata.labels["policy.cozystack.io/allow-to-apiserver"]
30+
value: "true"
31+
- equal:
32+
path: spec.template.spec.serviceAccountName
33+
value: monitoring-cleanup
34+
- equal:
35+
path: spec.template.spec.restartPolicy
36+
value: Never
37+
- equal:
38+
path: spec.template.spec.securityContext.runAsNonRoot
39+
value: true
40+
- equal:
41+
path: spec.template.spec.securityContext.runAsUser
42+
value: 65534
43+
- equal:
44+
path: spec.template.spec.containers[0].name
45+
value: cleanup
46+
- equal:
47+
path: spec.template.spec.containers[0].image
48+
value: docker.io/clastix/kubectl:v1.32
49+
- equal:
50+
path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
51+
value: true
52+
- contains:
53+
path: spec.template.spec.containers[0].securityContext.capabilities.drop
54+
content: ALL
55+
56+
- it: deletes the vm-operator storage PVCs by label
57+
documentSelector:
58+
path: kind
59+
value: Job
60+
asserts:
61+
- matchRegex:
62+
path: spec.template.spec.containers[0].command[2]
63+
pattern: kubectl delete pvc -n tenant-root -l managed-by=vm-operator --ignore-not-found
64+
65+
- it: deletes the cert-manager TLS secrets by name
66+
documentSelector:
67+
path: kind
68+
value: Job
69+
asserts:
70+
- matchRegex:
71+
path: spec.template.spec.containers[0].command[2]
72+
pattern: kubectl delete secret -n tenant-root alerta-tls grafana-ingress-tls --ignore-not-found
73+
74+
- it: creates the cleanup ServiceAccount as a post-delete hook
75+
documentSelector:
76+
path: kind
77+
value: ServiceAccount
78+
asserts:
79+
- equal:
80+
path: metadata.name
81+
value: monitoring-cleanup
82+
- equal:
83+
path: metadata.annotations["helm.sh/hook"]
84+
value: post-delete
85+
- equal:
86+
path: metadata.annotations["helm.sh/hook-weight"]
87+
value: "0"
88+
89+
- it: grants RBAC to delete PVCs by label and the two named TLS secrets
90+
documentSelector:
91+
path: kind
92+
value: Role
93+
asserts:
94+
- equal:
95+
path: metadata.name
96+
value: monitoring-cleanup
97+
- equal:
98+
path: metadata.annotations["helm.sh/hook-weight"]
99+
value: "5"
100+
- equal:
101+
path: rules[0].resources[0]
102+
value: persistentvolumeclaims
103+
- contains:
104+
path: rules[0].verbs
105+
content: delete
106+
- equal:
107+
path: rules[1].resources[0]
108+
value: secrets
109+
- equal:
110+
path: rules[1].resourceNames
111+
value:
112+
- alerta-tls
113+
- grafana-ingress-tls
114+
- contains:
115+
path: rules[1].verbs
116+
content: delete
117+
118+
- it: binds the cleanup Role to the cleanup ServiceAccount
119+
documentSelector:
120+
path: kind
121+
value: RoleBinding
122+
asserts:
123+
- equal:
124+
path: metadata.name
125+
value: monitoring-cleanup
126+
- equal:
127+
path: roleRef.name
128+
value: monitoring-cleanup
129+
- equal:
130+
path: subjects[0].name
131+
value: monitoring-cleanup

0 commit comments

Comments
 (0)