Skip to content

Commit 4d819e5

Browse files
committed
[api] Use shared informer cache
This patch changes all clients in the Cozystack API server to typed ones from the controller runtime. This should improve the performance of the API server and simplifies the code by removing work with unstructured objects and dynamic clients. ```release-note [api] Use typed and cache-backed k8s clients in the Cozystack API to improve performance. Get rid of operations on unstructured objects and use of dynamic clients. ``` Signed-off-by: Timofei Larkin <[email protected]>
1 parent 21de4f7 commit 4d819e5

6 files changed

Lines changed: 281 additions & 238 deletions

File tree

‎pkg/apiserver/apiserver.go‎

Lines changed: 61 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,22 @@ limitations under the License.
1717
package apiserver
1818

1919
import (
20+
"context"
2021
"fmt"
22+
"time"
2123

2224
helmv2 "github.com/fluxcd/helm-controller/api/v2"
25+
corev1 "k8s.io/api/core/v1"
26+
rbacv1 "k8s.io/api/rbac/v1"
2327
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2428
"k8s.io/apimachinery/pkg/runtime"
2529
"k8s.io/apimachinery/pkg/runtime/schema"
2630
"k8s.io/apimachinery/pkg/runtime/serializer"
2731
"k8s.io/apiserver/pkg/registry/rest"
2832
genericapiserver "k8s.io/apiserver/pkg/server"
29-
"k8s.io/client-go/dynamic"
30-
"k8s.io/client-go/kubernetes"
31-
restclient "k8s.io/client-go/rest"
33+
ctrl "sigs.k8s.io/controller-runtime"
34+
"sigs.k8s.io/controller-runtime/pkg/cache"
35+
"sigs.k8s.io/controller-runtime/pkg/client"
3236

3337
"github.com/cozystack/cozystack/pkg/apis/apps"
3438
appsinstall "github.com/cozystack/cozystack/pkg/apis/apps/install"
@@ -50,6 +54,7 @@ var (
5054
// versions and content types.
5155
Codecs = serializer.NewCodecFactory(Scheme)
5256
CozyComponentName = "cozy"
57+
syncPeriod = 5 * time.Minute
5358
)
5459

5560
func init() {
@@ -58,9 +63,15 @@ func init() {
5863

5964
// Register HelmRelease types.
6065
if err := helmv2.AddToScheme(Scheme); err != nil {
61-
panic(fmt.Sprintf("Failed to add HelmRelease types to scheme: %v", err))
66+
panic(fmt.Errorf("Failed to add HelmRelease types to scheme: %w", err))
6267
}
6368

69+
if err := corev1.AddToScheme(Scheme); err != nil {
70+
panic(fmt.Errorf("Failed to add core types to scheme: %w", err))
71+
}
72+
if err := rbacv1.AddToScheme(Scheme); err != nil {
73+
panic(fmt.Errorf("Failed to add RBAC types to scheme: %w", err))
74+
}
6475
// Add unversioned types.
6576
metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})
6677

@@ -118,43 +129,59 @@ func (c completedConfig) New() (*CozyServer, error) {
118129
}
119130

120131
// Create a dynamic client for HelmRelease using InClusterConfig.
121-
inClusterConfig, err := restclient.InClusterConfig()
132+
cfg, err := ctrl.GetConfig()
122133
if err != nil {
123-
return nil, fmt.Errorf("unable to get in-cluster config: %v", err)
134+
return nil, fmt.Errorf("failed to get kubeconfig: %w", err)
124135
}
125136

126-
dynamicClient, err := dynamic.NewForConfig(inClusterConfig)
137+
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
138+
Scheme: Scheme,
139+
Cache: cache.Options{SyncPeriod: &syncPeriod},
140+
})
127141
if err != nil {
128-
return nil, fmt.Errorf("unable to create dynamic client: %v", err)
142+
return nil, fmt.Errorf("failed to build manager: %w", err)
129143
}
130144

131-
clientset, err := kubernetes.NewForConfig(inClusterConfig)
132-
if err != nil {
133-
return nil, fmt.Errorf("create kube clientset: %v", err)
145+
ctx := ctrl.SetupSignalHandler()
146+
147+
if err = mustGetInformers(ctx, mgr,
148+
&helmv2.HelmRelease{},
149+
&corev1.Secret{},
150+
&corev1.Namespace{},
151+
&corev1.Service{},
152+
&rbacv1.RoleBinding{},
153+
); err != nil {
154+
return nil, fmt.Errorf("failed to get informers: %w", err)
155+
}
156+
157+
go func() {
158+
if err := mgr.Start(ctx); err != nil {
159+
panic(fmt.Errorf("manager start failed: %w", err))
160+
}
161+
}()
162+
163+
if ok := mgr.GetCache().WaitForCacheSync(ctx); !ok {
164+
return nil, fmt.Errorf("cache sync failed")
134165
}
135166

167+
cli := mgr.GetClient()
168+
watchCli, err := client.NewWithWatch(cfg, client.Options{Scheme: Scheme})
169+
if err != nil {
170+
return nil, fmt.Errorf("failed to build watch client: %w", err)
171+
}
136172
// --- static, cluster-scoped resource for core group ---
137173
coreV1alpha1Storage := map[string]rest.Storage{}
138174
coreV1alpha1Storage["tenantnamespaces"] = cozyregistry.RESTInPeace(
139-
tenantnamespacestorage.NewREST(
140-
clientset.CoreV1(),
141-
clientset.RbacV1(),
142-
),
175+
tenantnamespacestorage.NewREST(cli, watchCli),
143176
)
144177
coreV1alpha1Storage["tenantsecrets"] = cozyregistry.RESTInPeace(
145-
tenantsecretstorage.NewREST(
146-
clientset.CoreV1(),
147-
),
178+
tenantsecretstorage.NewREST(cli, watchCli),
148179
)
149180
coreV1alpha1Storage["tenantsecretstables"] = cozyregistry.RESTInPeace(
150-
tenantsecretstablestorage.NewREST(
151-
clientset.CoreV1(),
152-
),
181+
tenantsecretstablestorage.NewREST(cli, watchCli),
153182
)
154183
coreV1alpha1Storage["tenantmodules"] = cozyregistry.RESTInPeace(
155-
tenantmodulestorage.NewREST(
156-
dynamicClient,
157-
),
184+
tenantmodulestorage.NewREST(cli, watchCli),
158185
)
159186

160187
coreApiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(core.GroupName, Scheme, metav1.ParameterCodec, Codecs)
@@ -166,7 +193,7 @@ func (c completedConfig) New() (*CozyServer, error) {
166193
// --- dynamically-configured, per-tenant resources ---
167194
appsV1alpha1Storage := map[string]rest.Storage{}
168195
for _, resConfig := range c.ResourceConfig.Resources {
169-
storage := applicationstorage.NewREST(dynamicClient, &resConfig)
196+
storage := applicationstorage.NewREST(cli, watchCli, &resConfig)
170197
appsV1alpha1Storage[resConfig.Application.Plural] = cozyregistry.RESTInPeace(storage)
171198
}
172199
appsApiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apps.GroupName, Scheme, metav1.ParameterCodec, Codecs)
@@ -177,3 +204,12 @@ func (c completedConfig) New() (*CozyServer, error) {
177204

178205
return s, nil
179206
}
207+
208+
func mustGetInformers(ctx context.Context, mgr ctrl.Manager, types ...client.Object) error {
209+
for i := range types {
210+
if _, err := mgr.GetCache().GetInformer(ctx, types[i]); err != nil {
211+
return fmt.Errorf("failed to get informer for %T: %w", types[i], err)
212+
}
213+
}
214+
return nil
215+
}

0 commit comments

Comments
 (0)