Skip to content

Commit e1fdd79

Browse files
committed
rewrite CRD discovery to use openapi instead
On-behalf-of: @SAP [email protected]
1 parent 525e44a commit e1fdd79

File tree

5 files changed

+232
-36
lines changed

5 files changed

+232
-36
lines changed

cmd/crd-puller/main.go

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
Copyright 2025 The KCP Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package main
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"log"
23+
24+
"github.com/spf13/pflag"
25+
26+
"github.com/kcp-dev/api-syncagent/internal/discovery"
27+
28+
"k8s.io/apimachinery/pkg/runtime/schema"
29+
"k8s.io/client-go/tools/clientcmd"
30+
"sigs.k8s.io/yaml"
31+
)
32+
33+
var (
34+
kubeconfigPath string
35+
)
36+
37+
func main() {
38+
ctx := context.Background()
39+
40+
pflag.StringVar(&kubeconfigPath, "kubeconfig", "", "Path to the kubeconfig file to use (defaults to $KUBECONFIG)")
41+
pflag.Parse()
42+
43+
if pflag.NArg() == 0 {
44+
log.Fatal("No argument given. Please specify a GVK in the form 'Kind.version.apigroup.com' to pull.")
45+
}
46+
47+
gvk, _ := schema.ParseKindArg(pflag.Arg(0))
48+
if gvk == nil {
49+
log.Fatal("Invalid GVK, please use the format 'Kind.version.apigroup.com'.")
50+
}
51+
52+
loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
53+
loadingRules.ExplicitPath = kubeconfigPath
54+
55+
startingConfig, err := loadingRules.GetStartingConfig()
56+
if err != nil {
57+
log.Fatalf("Failed to load Kubernetes configuration: %v.", err)
58+
}
59+
60+
config, err := clientcmd.NewDefaultClientConfig(*startingConfig, nil).ClientConfig()
61+
if err != nil {
62+
log.Fatalf("Failed to load Kubernetes configuration: %v.", err)
63+
}
64+
65+
discoveryClient, err := discovery.NewClient(config)
66+
if err != nil {
67+
log.Fatalf("Failed to create discovery client: %v.", err)
68+
}
69+
70+
crd, err := discoveryClient.RetrieveCRD(ctx, *gvk)
71+
if err != nil {
72+
log.Fatalf("Failed to pull CRD: %v.", err)
73+
}
74+
75+
enc, err := yaml.Marshal(crd)
76+
if err != nil {
77+
log.Fatalf("Failed to encode CRD as YAML: %v.", err)
78+
}
79+
80+
fmt.Println(string(enc))
81+
}

internal/controller/apiresourceschema/controller.go

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323
"encoding/json"
2424
"fmt"
2525
"reflect"
26-
"slices"
2726
"strings"
2827

2928
"github.com/kcp-dev/logicalcluster/v3"
@@ -41,6 +40,7 @@ import (
4140
apierrors "k8s.io/apimachinery/pkg/api/errors"
4241
"k8s.io/apimachinery/pkg/labels"
4342
"k8s.io/apimachinery/pkg/types"
43+
"k8s.io/client-go/rest"
4444
"k8s.io/client-go/tools/record"
4545
"sigs.k8s.io/controller-runtime/pkg/builder"
4646
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -58,6 +58,7 @@ const (
5858
type Reconciler struct {
5959
localClient ctrlruntimeclient.Client
6060
kcpClient ctrlruntimeclient.Client
61+
restConfig *rest.Config
6162
log *zap.SugaredLogger
6263
recorder record.EventRecorder
6364
lcName logicalcluster.Name
@@ -79,6 +80,7 @@ func Add(
7980
reconciler := &Reconciler{
8081
localClient: mgr.GetClient(),
8182
kcpClient: kcpCluster.GetClient(),
83+
restConfig: mgr.GetConfig(),
8284
lcName: lcName,
8385
log: log.Named(ControllerName),
8486
recorder: mgr.GetEventRecorderFor(ControllerName),
@@ -127,7 +129,12 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, pubR
127129
// find the resource that the PublishedResource is referring to
128130
localGVK := projection.PublishedResourceSourceGVK(pubResource)
129131

130-
crd, err := discovery.NewClient(r.localClient).DiscoverResourceType(ctx, localGVK.GroupKind())
132+
client, err := discovery.NewClient(r.restConfig)
133+
if err != nil {
134+
return nil, fmt.Errorf("failed to create discovery client: %w", err)
135+
}
136+
137+
crd, err := client.RetrieveCRD(ctx, localGVK)
131138
if err != nil {
132139
return nil, fmt.Errorf("failed to discover resource defined in PublishedResource: %w", err)
133140
}
@@ -203,21 +210,9 @@ func (r *Reconciler) applyProjection(apiGroup string, crd *apiextensionsv1.Custo
203210
result := crd.DeepCopy()
204211
result.Spec.Group = apiGroup
205212

206-
// At this moment we ignore every non-selected version in the CRD, as we have not fully
207-
// decided on how to support the API version lifecycle yet. Having multiple versions in
208-
// the CRD will make kcp require a `conversion` to also be configured. Since we cannot
209-
// enforce that and want to instead work with existing CRDs as best as we can, we chose
210-
// this option (instead of error'ing out if a conversion is missing).
211-
result.Spec.Conversion = nil
212-
result.Spec.Versions = slices.DeleteFunc(result.Spec.Versions, func(v apiextensionsv1.CustomResourceDefinitionVersion) bool {
213-
return v.Name != pr.Spec.Resource.Version
214-
})
215-
216-
if len(result.Spec.Versions) != 1 {
217-
// This should never happen because of checks earlier in the reconciler.
218-
return nil, fmt.Errorf("invalid CRD: cannot find selected version %q", pr.Spec.Resource.Version)
219-
}
220-
213+
// Currently CRDs generated by our discovery mechanism already set these to true, but that's just
214+
// because it doesn't care to set them correctly; we keep this code here because from here on,
215+
// in kcp, we definitely want them to be true.
221216
result.Spec.Versions[0].Served = true
222217
result.Spec.Versions[0].Storage = true
223218

internal/controller/sync/controller.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ func Create(
9090
remoteDummy.SetGroupVersionKind(remoteGVK)
9191

9292
// find the local CRD so we know the actual local object scope
93-
localCRD, err := discoveryClient.DiscoverResourceType(ctx, localGVK.GroupKind())
93+
localCRD, err := discoveryClient.RetrieveCRD(ctx, localGVK)
9494
if err != nil {
9595
return nil, fmt.Errorf("failed to find local CRD: %w", err)
9696
}

internal/controller/syncmanager/controller.go

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,11 @@ func Add(
9898
stateNamespace string,
9999
agentName string,
100100
) error {
101+
discoveryClient, err := discovery.NewClient(localManager.GetConfig())
102+
if err != nil {
103+
return fmt.Errorf("failed to create discovery client: %w", err)
104+
}
105+
101106
reconciler := &Reconciler{
102107
ctx: ctx,
103108
localManager: localManager,
@@ -107,13 +112,13 @@ func Add(
107112
log: log,
108113
recorder: localManager.GetEventRecorderFor(ControllerName),
109114
syncWorkers: map[string]lifecycle.Controller{},
110-
discoveryClient: discovery.NewClient(localManager.GetClient()),
115+
discoveryClient: discoveryClient,
111116
prFilter: prFilter,
112117
stateNamespace: stateNamespace,
113118
agentName: agentName,
114119
}
115120

116-
_, err := builder.ControllerManagedBy(localManager).
121+
_, err = builder.ControllerManagedBy(localManager).
117122
Named(ControllerName).
118123
WithOptions(controller.Options{
119124
// this controller is meant to control others, so we only want 1 thread

internal/discovery/client.go

Lines changed: 131 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -19,39 +19,154 @@ package discovery
1919
import (
2020
"context"
2121
"fmt"
22+
"strings"
2223

24+
"github.com/kcp-dev/kcp/pkg/crdpuller"
25+
26+
"k8s.io/apiextensions-apiserver/pkg/apihelpers"
2327
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
28+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2429
"k8s.io/apimachinery/pkg/runtime/schema"
25-
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
30+
utilerrors "k8s.io/apimachinery/pkg/util/errors"
31+
"k8s.io/apimachinery/pkg/util/sets"
32+
"k8s.io/apiserver/pkg/endpoints/openapi"
33+
"k8s.io/client-go/discovery"
34+
"k8s.io/client-go/rest"
35+
"k8s.io/kube-openapi/pkg/util/proto"
2636
)
2737

2838
type Client struct {
29-
kubeClient ctrlruntimeclient.Reader
39+
discoveryClient discovery.DiscoveryInterface
3040
}
3141

32-
func NewClient(kubeClient ctrlruntimeclient.Client) *Client {
33-
return &Client{
34-
kubeClient: kubeClient,
42+
func NewClient(config *rest.Config) (*Client, error) {
43+
discoveryClient, err := discovery.NewDiscoveryClientForConfig(config)
44+
if err != nil {
45+
return nil, err
3546
}
47+
48+
return &Client{
49+
discoveryClient: discoveryClient,
50+
}, nil
3651
}
3752

38-
func (c *Client) DiscoverResourceType(ctx context.Context, gk schema.GroupKind) (*apiextensionsv1.CustomResourceDefinition, error) {
39-
crds := &apiextensionsv1.CustomResourceDefinitionList{}
40-
if err := c.kubeClient.List(ctx, crds); err != nil {
41-
return nil, fmt.Errorf("failed to list CRDs: %w", err)
53+
func (c *Client) RetrieveCRD(ctx context.Context, gvk schema.GroupVersionKind) (*apiextensionsv1.CustomResourceDefinition, error) {
54+
openapiSchema, err := c.discoveryClient.OpenAPISchema()
55+
if err != nil {
56+
return nil, err
57+
}
58+
59+
// Most of this code follows the logic in kcp's crd-puller, but is slimmed down
60+
// to a) only support openapi and b) extract a specific version, not necessarily
61+
// the preferred version.
62+
63+
models, err := proto.NewOpenAPIData(openapiSchema)
64+
if err != nil {
65+
return nil, err
66+
}
67+
modelsByGKV, err := openapi.GetModelsByGKV(models)
68+
if err != nil {
69+
return nil, err
4270
}
4371

44-
for _, crd := range crds.Items {
45-
if crd.Spec.Group != gk.Group {
46-
continue
72+
protoSchema := modelsByGKV[gvk]
73+
if protoSchema == nil {
74+
return nil, fmt.Errorf("no models for %v", gvk)
75+
}
76+
77+
var schemaProps apiextensionsv1.JSONSchemaProps
78+
errs := crdpuller.Convert(protoSchema, &schemaProps)
79+
if len(errs) > 0 {
80+
return nil, utilerrors.NewAggregate(errs)
81+
}
82+
83+
_, resourceLists, err := c.discoveryClient.ServerGroupsAndResources()
84+
if err != nil {
85+
return nil, err
86+
}
87+
88+
var resource *metav1.APIResource
89+
allResourceNames := sets.New[string]()
90+
for _, resList := range resourceLists {
91+
for _, res := range resList.APIResources {
92+
allResourceNames.Insert(res.Name)
93+
94+
// find the requested resource based on the Kind, but ensure that subresources
95+
// are not misinterpreted as the main resource by checking for "/"
96+
if resList.GroupVersion == gvk.GroupVersion().String() && res.Kind == gvk.Kind && !strings.Contains(res.Name, "/") {
97+
resource = &res
98+
}
4799
}
100+
}
101+
102+
if resource == nil {
103+
return nil, fmt.Errorf("could not find %v in APIs", gvk)
104+
}
105+
106+
hasSubResource := func(subResource string) bool {
107+
return allResourceNames.Has(resource.Name + "/" + subResource)
108+
}
109+
110+
var statusSubResource *apiextensionsv1.CustomResourceSubresourceStatus
111+
if hasSubResource("status") {
112+
statusSubResource = &apiextensionsv1.CustomResourceSubresourceStatus{}
113+
}
48114

49-
if crd.Spec.Names.Kind != gk.Kind {
50-
continue
115+
var scaleSubResource *apiextensionsv1.CustomResourceSubresourceScale
116+
if hasSubResource("scale") {
117+
scaleSubResource = &apiextensionsv1.CustomResourceSubresourceScale{
118+
SpecReplicasPath: ".spec.replicas",
119+
StatusReplicasPath: ".status.replicas",
51120
}
121+
}
122+
123+
scope := apiextensionsv1.ClusterScoped
124+
if resource.Namespaced {
125+
scope = apiextensionsv1.NamespaceScoped
126+
}
52127

53-
return &crd, nil
128+
crd := &apiextensionsv1.CustomResourceDefinition{
129+
TypeMeta: metav1.TypeMeta{
130+
Kind: "CustomResourceDefinition",
131+
APIVersion: "apiextensions.k8s.io/v1",
132+
},
133+
ObjectMeta: metav1.ObjectMeta{
134+
Name: fmt.Sprintf("%s.%s", resource.Name, gvk.Group),
135+
},
136+
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
137+
Group: gvk.Group,
138+
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
139+
{
140+
Name: gvk.Version,
141+
Schema: &apiextensionsv1.CustomResourceValidation{
142+
OpenAPIV3Schema: &schemaProps,
143+
},
144+
Subresources: &apiextensionsv1.CustomResourceSubresources{
145+
Status: statusSubResource,
146+
Scale: scaleSubResource,
147+
},
148+
Served: true,
149+
Storage: true,
150+
},
151+
},
152+
Scope: scope,
153+
Names: apiextensionsv1.CustomResourceDefinitionNames{
154+
Plural: resource.Name,
155+
Kind: resource.Kind,
156+
Categories: resource.Categories,
157+
ShortNames: resource.ShortNames,
158+
Singular: resource.SingularName,
159+
},
160+
},
161+
}
162+
163+
apiextensionsv1.SetDefaults_CustomResourceDefinition(crd)
164+
165+
if apihelpers.IsProtectedCommunityGroup(gvk.Group) {
166+
crd.Annotations = map[string]string{
167+
apiextensionsv1.KubeAPIApprovedAnnotation: "https://github.com/kcp-dev/kubernetes/pull/4",
168+
}
54169
}
55170

56-
return nil, fmt.Errorf("CustomResourceDefinition for %s/%s does not exist", gk.Group, gk.Kind)
171+
return crd, nil
57172
}

0 commit comments

Comments
 (0)