Skip to content

Commit 246e0fc

Browse files
committed
[cozystack-controller] Ancestor tracking webhook
Many resources created as part of managed apps in cozystack (pods, secrets, etc) do not carry predictable labels that unambiguously indicate which app originally triggered their creation. Some resources are managed by controllers and other custom resources and this indirection can lead to loss of information. Other controllers sometimes simply do not allow setting labels on controlled resources and the latter do not inherit labels from the owner. This patch implements a webhook that sidesteps this problem with a universal solution. On creation of a pod/secret/PVC etc it walks through the owner references until a HelmRelease is found that can be matched with a managed app dynamically registered in the Cozystack API server. The pod is mutated with labels identifying the managed app. ```release-note [cozystack-controller] Add a mutating webhook to identify the Cozystack managed app that ultimately owns low-level resources created in the cluster and label these resources with a reference to said app. ``` Signed-off-by: Timofei Larkin <[email protected]>
1 parent 9f89ef3 commit 246e0fc

8 files changed

Lines changed: 554 additions & 0 deletions

File tree

‎cmd/cozystack-controller/main.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ import (
3838

3939
cozystackiov1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
4040
"github.com/cozystack/cozystack/internal/controller"
41+
lcw "github.com/cozystack/cozystack/internal/lineagecontrollerwebhook"
4142
"github.com/cozystack/cozystack/internal/telemetry"
4243

4344
helmv2 "github.com/fluxcd/helm-controller/api/v2"
@@ -214,6 +215,20 @@ func main() {
214215
os.Exit(1)
215216
}
216217

218+
// special one that's both a webhook and a reconciler
219+
lineageControllerWebhook := &lcw.LineageControllerWebhook{
220+
Client: mgr.GetClient(),
221+
Scheme: mgr.GetScheme(),
222+
}
223+
if err := lineageControllerWebhook.SetupWithManagerAsController(mgr); err != nil {
224+
setupLog.Error(err, "unable to setup controller", "controller", "LineageController")
225+
os.Exit(1)
226+
}
227+
if err := lineageControllerWebhook.SetupWithManagerAsWebhook(mgr); err != nil {
228+
setupLog.Error(err, "unable to setup webhook", "webhook", "LineageWebhook")
229+
os.Exit(1)
230+
}
231+
217232
// +kubebuilder:scaffold:builder
218233

219234
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package lineagecontrollerwebhook
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
7+
helmv2 "github.com/fluxcd/helm-controller/api/v2"
8+
)
9+
10+
type runtimeConfig struct {
11+
chartAppMap map[string]string
12+
}
13+
14+
// not threadsafe, btw
15+
func (l *LineageControllerWebhook) initConifg() {
16+
if l.config.Load() == nil {
17+
l.config.Store(&runtimeConfig{chartAppMap: make(map[string]string)})
18+
}
19+
}
20+
21+
func (l *LineageControllerWebhook) Map(hr *helmv2.HelmRelease) (string, string, string, error) {
22+
cfg := l.config.Load().(*runtimeConfig).chartAppMap
23+
val, ok := cfg[hr.Spec.Chart.Spec.SourceRef.Name+"/"+hr.Spec.Chart.Spec.Chart]
24+
if !ok {
25+
return "", "", "", fmt.Errorf("cannot map helm release %s/%s to dynamic app", hr.Namespace, hr.Name)
26+
}
27+
split := strings.Split(val, "/")
28+
return strings.Join(split[:2], "/"), split[2], split[3], nil
29+
}
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package lineagecontrollerwebhook
2+
3+
import (
4+
"context"
5+
"fmt"
6+
7+
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
8+
ctrl "sigs.k8s.io/controller-runtime"
9+
"sigs.k8s.io/controller-runtime/pkg/client"
10+
"sigs.k8s.io/controller-runtime/pkg/log"
11+
)
12+
13+
func (c *LineageControllerWebhook) SetupWithManagerAsController(mgr ctrl.Manager) error {
14+
return ctrl.NewControllerManagedBy(mgr).
15+
For(&cozyv1alpha1.CozystackResourceDefinition{}).
16+
Complete(c)
17+
}
18+
19+
func (c *LineageControllerWebhook) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
20+
l := log.FromContext(ctx)
21+
crds := &cozyv1alpha1.CozystackResourceDefinitionList{}
22+
if err := c.List(ctx, crds, &client.ListOptions{Namespace: "cozy-system"}); err != nil {
23+
l.Error(err, "failed reading CozystackResourceDefinitions")
24+
}
25+
newConfig := make(map[string]string)
26+
for _, crd := range crds.Items {
27+
k := fmt.Sprintf("%s/%s", crd.Spec.Release.Chart.SourceRef.Name, crd.Spec.Release.Chart.Name)
28+
v := fmt.Sprintf("%s/%s/%s", "apps.cozystack.io/v1alpha1", crd.Spec.Application.Kind, crd.Spec.Release.Prefix)
29+
newConfig[k] = v
30+
}
31+
c.config.Store(&runtimeConfig{newConfig})
32+
return ctrl.Result{}, nil
33+
}
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package lineagecontrollerwebhook
2+
3+
import (
4+
"sync/atomic"
5+
6+
"k8s.io/apimachinery/pkg/api/meta"
7+
"k8s.io/apimachinery/pkg/runtime"
8+
"k8s.io/client-go/dynamic"
9+
"sigs.k8s.io/controller-runtime/pkg/client"
10+
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
11+
)
12+
13+
// +kubebuilder:webhook:path=/mutate-v1-pod,mutating=true,failurePolicy=Fail,sideEffects=None,groups="",resources=pods,verbs=create;update,versions=v1,name=mpods.cozystack.io,admissionReviewVersions={v1}
14+
type LineageControllerWebhook struct {
15+
Scheme *runtime.Scheme
16+
client.Client
17+
decoder admission.Decoder
18+
dynClient dynamic.Interface
19+
mapper meta.RESTMapper
20+
config atomic.Value
21+
}
Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
package lineagecontrollerwebhook
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"errors"
7+
"fmt"
8+
9+
"github.com/cozystack/cozystack/pkg/lineage"
10+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
11+
"k8s.io/apimachinery/pkg/runtime/schema"
12+
"k8s.io/client-go/discovery"
13+
"k8s.io/client-go/discovery/cached/memory"
14+
"k8s.io/client-go/dynamic"
15+
"k8s.io/client-go/rest"
16+
"k8s.io/client-go/restmapper"
17+
ctrl "sigs.k8s.io/controller-runtime"
18+
"sigs.k8s.io/controller-runtime/pkg/client"
19+
"sigs.k8s.io/controller-runtime/pkg/log"
20+
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
21+
)
22+
23+
var (
24+
NoAncestors = fmt.Errorf("no managed apps found in lineage")
25+
AncestryAmbiguous = fmt.Errorf("object ancestry is ambiguous")
26+
)
27+
28+
// SetupWithManager registers the handler with the webhook server.
29+
func (h *LineageControllerWebhook) SetupWithManagerAsWebhook(mgr ctrl.Manager) error {
30+
cfg := rest.CopyConfig(mgr.GetConfig())
31+
32+
var err error
33+
h.dynClient, err = dynamic.NewForConfig(cfg)
34+
if err != nil {
35+
return err
36+
}
37+
38+
discoClient, err := discovery.NewDiscoveryClientForConfig(cfg)
39+
if err != nil {
40+
return err
41+
}
42+
43+
cachedDisco := memory.NewMemCacheClient(discoClient)
44+
h.mapper = restmapper.NewDeferredDiscoveryRESTMapper(cachedDisco)
45+
46+
// Register HTTP path -> handler.
47+
mgr.GetWebhookServer().Register("/mutate-lineage", &admission.Webhook{Handler: h})
48+
49+
return nil
50+
}
51+
52+
// InjectDecoder lets controller-runtime give us a decoder for AdmissionReview requests.
53+
func (h *LineageControllerWebhook) InjectDecoder(d admission.Decoder) error {
54+
h.decoder = d
55+
return nil
56+
}
57+
58+
// Handle is called for each AdmissionReview that matches the webhook config.
59+
func (h *LineageControllerWebhook) Handle(ctx context.Context, req admission.Request) admission.Response {
60+
logger := log.FromContext(ctx).WithValues(
61+
"gvk", req.Kind.String(),
62+
"namespace", req.Namespace,
63+
"name", req.Name,
64+
"operation", req.Operation,
65+
)
66+
warn := make(admission.Warnings, 0)
67+
68+
obj := &unstructured.Unstructured{}
69+
if err := h.decodeUnstructured(req, obj); err != nil {
70+
return admission.Errored(400, fmt.Errorf("decode object: %w", err))
71+
}
72+
73+
labels, err := h.computeLabels(ctx, obj)
74+
for {
75+
if err != nil && errors.Is(err, NoAncestors) {
76+
return admission.Allowed("object not managed by app")
77+
}
78+
if err != nil && errors.Is(err, AncestryAmbiguous) {
79+
warn = append(warn, "object ancestry ambiguous, using first ancestor found")
80+
break
81+
}
82+
if err != nil {
83+
return admission.Errored(500, fmt.Errorf("error computing lineage labels: %w", err))
84+
}
85+
if err == nil {
86+
break
87+
}
88+
}
89+
90+
h.applyLabels(obj, labels)
91+
92+
mutated, err := json.Marshal(obj)
93+
if err != nil {
94+
return admission.Errored(500, fmt.Errorf("marshal mutated pod: %w", err))
95+
}
96+
logger.V(1).Info("mutated pod", "namespace", obj.GetNamespace(), "name", obj.GetName())
97+
return admission.PatchResponseFromRaw(req.Object.Raw, mutated).WithWarnings(warn...)
98+
}
99+
100+
func (h *LineageControllerWebhook) computeLabels(ctx context.Context, o *unstructured.Unstructured) (map[string]string, error) {
101+
var err error
102+
owners := lineage.WalkOwnershipGraph(ctx, h.dynClient, h.mapper, h, o)
103+
if len(owners) == 0 {
104+
return nil, NoAncestors
105+
}
106+
if len(owners) > 1 {
107+
err = AncestryAmbiguous
108+
}
109+
obj, err2 := owners[0].GetUnstructured(ctx, h.dynClient, h.mapper)
110+
if err2 != nil {
111+
return nil, err2
112+
}
113+
gv, err := schema.ParseGroupVersion(obj.GetAPIVersion())
114+
if err != nil {
115+
// should never happen, we got an APIVersion right from the API
116+
return nil, fmt.Errorf("could not parse APIVersion %s to a group and version: %w", obj.GetAPIVersion(), err)
117+
}
118+
return map[string]string{
119+
// truncate apigroup to first 63 chars
120+
"apps.cozystack.io/application.group": func(s string) string {
121+
if len(s) < 63 {
122+
return s
123+
}
124+
return s[:63]
125+
}(gv.Group),
126+
"apps.cozystack.io/application.kind": obj.GetKind(),
127+
"apps.cozystack.io/application.name": obj.GetName(),
128+
}, err
129+
}
130+
131+
func (h *LineageControllerWebhook) applyLabels(o client.Object, labels map[string]string) {
132+
existing := o.GetLabels()
133+
if existing == nil {
134+
existing = make(map[string]string)
135+
}
136+
for k, v := range labels {
137+
existing[k] = v
138+
}
139+
o.SetLabels(existing)
140+
}
141+
142+
func (h *LineageControllerWebhook) decodeUnstructured(req admission.Request, out *unstructured.Unstructured) error {
143+
if h.decoder != nil {
144+
if err := h.decoder.Decode(req, out); err == nil {
145+
return nil
146+
}
147+
if req.Kind.Group != "" || req.Kind.Kind != "" || req.Kind.Version != "" {
148+
out.SetGroupVersionKind(schema.GroupVersionKind{
149+
Group: req.Kind.Group,
150+
Version: req.Kind.Version,
151+
Kind: req.Kind.Kind,
152+
})
153+
if err := h.decoder.Decode(req, out); err == nil {
154+
return nil
155+
}
156+
}
157+
}
158+
if len(req.Object.Raw) == 0 {
159+
return errors.New("empty admission object")
160+
}
161+
return json.Unmarshal(req.Object.Raw, &out.Object)
162+
}

0 commit comments

Comments
 (0)