@@ -10,15 +10,25 @@ spec:
     namespace: cozy-system
     path: /
   variants:
-  # The aggregated API server for sdn.cozystack.io (VPC tenancy). Separate from
-  # the cozyplane CNI component on purpose: the CNI installs before everything
-  # (including cert-manager), while this needs cert-manager Certificates and the
-  # etcd-operator. Until it installs, the group is served by the CNI package's
-  # bootstrap CRDs; its APIService takes the group over atomically when it lands.
+  # The aggregated API server for sdn.cozystack.io (VPC tenancy, policy). It is a
+  # separate component on purpose: the CNI installs before everything (including
+  # cert-manager), while this needs cert-manager Certificates, the etcd-operator
+  # and — since its etcd is PVC-backed — storage.
+  #
+  # It is the ONLY server of the group: the tenant kinds have no CRDs. Until this
+  # component lands, sdn.cozystack.io simply is not served, and the default
+  # network keeps working regardless (the CNI's own CRDs are the separate LOCAL
+  # group, local.sdn.cozystack.io — underlay IPAM, which must come up beneath
+  # cert-manager, etcd and this server, all of which run as pods).
   - name: default
     dependsOn:
     - cozystack.cert-manager
     - cozystack.etcd-operator
+    # Its etcd is PVC-backed now (the aggregated API's objects are cluster
+    # state; a memory-backed etcd loses every VPC, Port and policy on a
+    # full-cluster restart — which is what a node reboot is), so the storage
+    # layer must exist first. No cycle: linstor depends on the CNI, not on this.
+    - cozystack.linstor
     components:
     - name: cozyplane-apiserver
       path: system/cozyplane-apiserver
 
@@ -175,10 +175,12 @@
 {{- $controllerValues := dict "cozystackController" (dict "disableTelemetry" true) -}}
 {{- $_ := set $cozystackEngineComponents "cozystack-controller" (dict "values" $controllerValues) -}}
 {{- end -}}
-{{- /* cozyplane owns the sdn.cozystack.io group (bootstrap CRDs, then the
-       cozyplane-apiserver aggregated server takes it over), so cozystack-api must
-       not aggregate it — a second APIService for the group would hijack it and
-       every non-SecurityGroup kind (VPC, Port, …) 404s. */ -}}
+{{- /* cozyplane owns the sdn.cozystack.io group — served ONLY by the
+       cozyplane-apiserver aggregated server; it has no CRDs (a group served by
+       both a CRD and an APIService makes the kube-apiserver's OpenAPI merge
+       collide on duplicated paths, and the group's schema then never serves).
+       So cozystack-api must not aggregate it: a second APIService for the group
+       would hijack it and every non-SecurityGroup kind (VPC, Port, …) 404s. */ -}}
 {{- if eq (.Values.bundles.system.networkingVariant | default "kubeovn-cilium") "cozyplane" -}}
 {{- $apiValues := index $cozystackEngineComponents "cozystack-api" | default (dict "values" (dict)) -}}
 {{- $apiInner := index $apiValues "values" | default (dict) -}}
@@ -204,10 +206,12 @@
        Cilium variants the cilium chart provides the CRDs, so skip it there. */ -}}
 {{- if eq (.Values.bundles.system.networkingVariant | default "kubeovn-cilium") "cozyplane" }}
 {{include "cozystack.platform.package.default" (list "cozystack.cozyplane-cilium-crds" $) }}
-{{- /* The aggregated sdn.cozystack.io apiserver: a separate component precisely so
-       the CNI slot stays cert-manager-free — this one dependsOn cert-manager +
-       etcd-operator (see sources/cozyplane-apiserver.yaml). Its APIService takes
-       the group over from the CNI package's bootstrap CRDs when it lands. */ -}}
+{{- /* The aggregated sdn.cozystack.io apiserver: a separate component precisely
+       so the CNI slot stays cert-manager-free — this one dependsOn cert-manager,
+       etcd-operator and (for its PVC-backed etcd) storage. There is no takeover:
+       the tenant kinds have no CRDs at all. The CNI package ships only the LOCAL
+       group's CRDs (local.sdn.cozystack.io — underlay IPAM, which must work
+       before cert-manager, etcd and this server, all of which are pods). */ -}}
 {{include "cozystack.platform.package.default" (list "cozystack.cozyplane-apiserver" $) }}
 {{- end }}
 {{include "cozystack.platform.package.default" (list "cozystack.cert-manager" $) }}
 
@@ -74,6 +74,13 @@ rules:
   - apiGroups: ["apiregistration.k8s.io"]
     resources: ["apiservices"]
     verbs: ["get", "create", "patch"]
+  # ...and then removes the bootstrap CRDs it superseded. They are shadowed for
+  # routing but NOT for OpenAPI: while they exist, the group's spec fails to
+  # merge (duplicated paths) and `kubectl apply` of any object in it dies with
+  # "failed to download openapi".
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["get", "list", "delete"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -91,8 +98,28 @@ subjects:
     namespace: {{ $ns }}
 ---
 {{- if not .Values.etcd.operator.enabled }}
-# Built-in single-pod etcd (dev/evaluation; ephemeral by default). Production
-# uses the operator-managed EtcdCluster instead — see templates/etcd-cluster.yaml.
+{{- if .Values.etcd.persistence.enabled }}
+# The single-pod etcd's storage. The aggregated API's objects are cluster state:
+# an emptyDir would lose every VPC, Port and policy the moment the pod moves.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: cozyplane-etcd-data
+  namespace: {{ $ns }}
+  labels:
+    {{- include "cozyplane-apiserver.labels" . | nindent 4 }}
+spec:
+  accessModes: [ReadWriteOnce]
+  {{- with .Values.etcd.persistence.storageClassName }}
+  storageClassName: {{ . | quote }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.etcd.persistence.size }}
+---
+{{- end }}
+# Built-in single-pod etcd. Production uses the operator-managed EtcdCluster
+# instead — see templates/etcd-cluster.yaml.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -103,6 +130,10 @@ metadata:
     {{- include "cozyplane-apiserver.labels" . | nindent 4 }}
 spec:
   replicas: 1
+  # Recreate, not RollingUpdate: two etcd pods must never hold the same RWO
+  # volume, and a rolling update would try.
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       app: cozyplane-etcd
@@ -244,9 +275,11 @@ spec:
             - --ensure-apiservice-service={{ $ns }}/cozyplane-apiserver
             - --ensure-apiservice-ca-injection={{ $ns }}/cozyplane-apiserver-serving
             {{- if .Values.etcd.operator.enabled }}
-            # The v1alpha2 operator's client Service (members carry
-            # operator-assigned names, so dial the Service, not per-member DNS).
-            - --etcd-servers=https://cozyplane-etcd.{{ $ns }}.svc:2379
+            {{- $eps := list }}
+            {{- range $i := until (int .Values.etcd.operator.replicas) }}
+            {{- $eps = append $eps (printf "https://cozyplane-etcd-%d.cozyplane-etcd-headless.%s.svc:2379" $i $ns) }}
+            {{- end }}
+            - --etcd-servers={{ join "," $eps }}
             - --etcd-cafile=/etc/etcd-certs/ca.crt
             - --etcd-certfile=/etc/etcd-certs/tls.crt
             - --etcd-keyfile=/etc/etcd-certs/tls.key
 
@@ -113,9 +113,19 @@ spec:
   version: {{ .Values.etcd.operator.version | quote }}
   replicas: {{ $replicas }}
   storage:
-    # Memory-backed (see the file header). `medium` is immutable post-create;
-    # size is the tmpfs cap, not a reservation. Non-zero is required for Memory.
+    {{- if .Values.etcd.operator.persistent }}
+    # PVC-backed: the aggregated API's objects (VPCs, Ports, policies) are
+    # cluster state. A memory-backed etcd loses every one of them on a
+    # full-cluster restart — which is exactly what a node reboot is.
+    medium: ""
+    {{- with .Values.etcd.operator.storageClass }}
+    storageClassName: {{ . | quote }}
+    {{- end }}
+    {{- else }}
+    # tmpfs: no storage layer needed, but NOT durable across a full-cluster
+    # restart. `medium` is immutable post-create; size is the tmpfs cap.
     medium: Memory
+    {{- end }}
     size: {{ .Values.etcd.operator.size }}
   options:
     autoCompactionMode: periodic
 
@@ -1,6 +1,7 @@
-# Container image (shared with the cozyplane package — the apiserver binary
-# lives in the same image). Digest-pinned; update on each published build.
-image: ghcr.io/lllamnyp/cozyplane@sha256:3d6fece3355f4f0accf47c4ab02d9b75919f002db4dee444a08f157d888c2a7b
+# Container image (shared with the cozyplane chart — the apiserver binary lives
+# in the same image). Digest-pinned (no floating tag); update on each published
+# build.
+image: ghcr.io/lllamnyp/cozyplane@sha256:c3858ecc3d590b05de81599f65bf2ed1ecd4708f41f6be80244ddd566b303101
 imagePullPolicy: IfNotPresent
 imagePullSecrets: []
 
@@ -13,28 +14,39 @@ resources:
     memory: 256Mi
 
 etcd:
-  # Operator-managed etcd (etcd-operator.cozystack.io/v1alpha2; Cozystack ships
-  # it): replicated, TLS end to end via the operator's cert-manager
-  # integration. MEMORY-BACKED (no PVC/storageClass) so the API's etcd needs no
-  # storage layer — this component installs after cert-manager but before (or
-  # independent of) Linstor. Trade-off: not durable across a full-cluster
-  # restart; a persistent (storageClassName) mode is a follow-up once storage
-  # is sequenced after the CNI.
+  # Operator-managed etcd (the aenix-io etcd-operator; Cozystack ships it):
+  # replicated, PVC-backed, TLS end to end. The production path — without it
+  # the built-in single-pod etcd below is used, and with its default
+  # emptyDir storage a single pod reschedule erases every sdn.cozystack.io
+  # object in the cluster.
   operator:
     enabled: true
-    version: "3.6.11"
     replicas: 3
-    # Memory-backed: `size` is the tmpfs cap (not a reservation), so keep it
-    # modest — the sdn.cozystack.io object set is small.
-    size: 1Gi
-    # StorageClass for a future PVC-backed mode ("" = cluster default);
-    # ignored while storage.medium=Memory.
-    storageClass: ""
-  # Built-in single-pod etcd (dev/evaluation only; unused while the operator
-  # mode above is enabled).
+    size: 4Gi
+    # persistent: PVC-backed (durable) vs tmpfs. The aggregated API's objects
+    # are cluster state; a memory-backed etcd loses all of them on a
+    # full-cluster restart, which is what a node reboot is. Requires a storage
+    # layer, so the component must be sequenced after it.
+    persistent: true
+    # StorageClass for the members' PVCs ("" = cluster default). Replicated, so
+    # a member's data survives losing the node it was scheduled on.
+    storageClass: "replicated"
+  # Built-in single-pod etcd (dev/evaluation only).
   image: registry.k8s.io/etcd:3.5.16-0
+  # Storage for the built-in etcd. The default is ephemeral (emptyDir); use a
+  # persistentVolumeClaim volume source to survive a pod restart.
+  # Storage for the built-in single-pod etcd. Default is a PVC — the aggregated
+  # API's objects (VPCs, Ports, policies) are cluster state, and an emptyDir
+  # loses them on a pod move. Set `emptyDir: {}` explicitly for a throwaway
+  # cluster with no storage class.
   storage:
-    emptyDir: {}
+    persistentVolumeClaim:
+      claimName: cozyplane-etcd-data
+  # The PVC the chart creates when storage above references it.
+  persistence:
+    enabled: true
+    storageClassName: ""   # "" = the cluster default
+    size: 2Gi
   resources:
     requests:
       cpu: 50m
 
@@ -20,10 +20,20 @@ rules:
     resources: ["nodes"]
     verbs: ["get", "list", "watch", "patch"]
   # The plugin (using this SA) reads pod annotations to resolve VPC membership,
-  # and the agent reads pods to distinguish revocation from ordinary teardown.
+  # the agent reads pods to distinguish revocation from ordinary teardown, and
+  # the NetworkPolicy compiler watches them for identities/membership
+  # (docs/network-policy.md).
   - apiGroups: [""]
     resources: ["pods"]
-    verbs: ["get"]
+    verbs: ["get", "list", "watch"]
+  # Namespace labels resolve namespaceSelector peers; NetworkPolicies are the
+  # net-0 policy source (docs/network-policy.md).
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["networkpolicies"]
+    verbs: ["get", "list", "watch"]
   # Tenant network ids from VPC objects (namespaced; read across all namespaces).
   - apiGroups: ["sdn.cozystack.io"]
     resources: ["vpcs"]
@@ -48,11 +58,27 @@ rules:
   - apiGroups: ["sdn.cozystack.io"]
     resources: ["securitygroups"]
     verbs: ["get", "list", "watch"]
+  # FabricIPs: the CNI plugin claims and releases underlay addresses through
+  # this SA (the agent writes its kubeconfig), and the agent watches them.
+  - apiGroups: ["local.sdn.cozystack.io"]
+    resources: ["fabricips"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  # HostFirewalls: the agent compiles the objects selecting its node into the
+  # hf_allow map and arms enforcement (docs/host-firewall.md).
+  - apiGroups: ["sdn.cozystack.io"]
+    resources: ["hostfirewalls"]
+    verbs: ["get", "list", "watch"]
   # FloatingIPs: the agent programs the floating map and advertises the address
   # for those whose target Port is on this node.
   - apiGroups: ["sdn.cozystack.io"]
     resources: ["floatingips"]
     verbs: ["get", "list", "watch"]
+  # ExternalPools: the agent ensures the datapath serves every pool's link on
+  # every node — LB/NodePort frontends arrive wherever the provider attracts
+  # them, and a DSR reply must leave by that link (docs/lb-ingress.md).
+  - apiGroups: ["sdn.cozystack.io"]
+    resources: ["externalpools"]
+    verbs: ["get", "list", "watch"]
   # ServiceVIPs: the agent programs the svc_vips datapath map from them, the
   # resolver answers attached ClusterIP services with the VIP, and the CNI
   # plugin counts them as used addresses when claiming a Port IP.
 
@@ -93,6 +93,11 @@ rules:
   - apiGroups: [""]
     resources: ["pods"]
     verbs: ["get", "list", "watch", "delete"]
+  # FabricIPs: the controller reclaims an underlay address whose pod is gone —
+  # the GC the host-local file store never had.
+  - apiGroups: ["local.sdn.cozystack.io"]
+    resources: ["fabricips"]
+    verbs: ["get", "list", "watch", "delete"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding