Skip to content

Commit 40bac3c

Browse files
pierDipimatzew
pierDipi
and
matzew
authored
Eventing TLS: add tests for dynamically added CA trust bundles (#7726)
* Add CA rotation tests Signed-off-by: Pierangelo Di Pilato <[email protected]> * Fix test Signed-off-by: Pierangelo Di Pilato <[email protected]> * Add propagation improvements for webhook (running before reconciler) Signed-off-by: Pierangelo Di Pilato <[email protected]> * Inject Kubeclient Signed-off-by: Pierangelo Di Pilato <[email protected]> * Fix unit tests Signed-off-by: Pierangelo Di Pilato <[email protected]> * Use managed T for ApiServerSource TLS tests Signed-off-by: Pierangelo Di Pilato <[email protected]> * Reduce number of retries Signed-off-by: Pierangelo Di Pilato <[email protected]> * 💄 updates after rebase... Signed-off-by: Matthias Wessendorf <[email protected]> --------- Signed-off-by: Pierangelo Di Pilato <[email protected]> Signed-off-by: Matthias Wessendorf <[email protected]> Co-authored-by: Matthias Wessendorf <[email protected]>
1 parent a2cf308 commit 40bac3c

File tree

14 files changed

+342
-19
lines changed

14 files changed

+342
-19
lines changed

‎pkg/apis/sources/v1/sinkbinding_lifecycle.go

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"strings"
2424

2525
"go.uber.org/zap"
26+
"k8s.io/client-go/kubernetes"
2627
corev1listers "k8s.io/client-go/listers/core/v1"
2728

2829
corev1 "k8s.io/api/core/v1"
@@ -214,13 +215,30 @@ func (sb *SinkBinding) Do(ctx context.Context, ps *duckv1.WithPod) {
214215
Value: ceOverrides,
215216
})
216217
}
217-
218-
pss, err := eventingtls.AddTrustBundleVolumes(GetTrustBundleConfigMapLister(ctx), sb, &ps.Spec.Template.Spec)
218+
gvk := schema.GroupVersionKind{
219+
Group: SchemeGroupVersion.Group,
220+
Version: SchemeGroupVersion.Version,
221+
Kind: "SinkBinding",
222+
}
223+
bundles, err := eventingtls.PropagateTrustBundles(ctx, getKubeClient(ctx), GetTrustBundleConfigMapLister(ctx), gvk, sb)
219224
if err != nil {
220-
logging.FromContext(ctx).Errorw("Failed to add trust bundle volumes %s/%s: %+v", zap.Error(err))
221-
return
225+
logging.FromContext(ctx).Errorw("Failed to propagate trust bundles", zap.Error(err))
226+
}
227+
if len(bundles) > 0 {
228+
pss, err := eventingtls.AddTrustBundleVolumesFromConfigMaps(bundles, &ps.Spec.Template.Spec)
229+
if err != nil {
230+
logging.FromContext(ctx).Errorw("Failed to add trust bundle volumes from configmaps %s/%s: %+v", zap.Error(err))
231+
return
232+
}
233+
ps.Spec.Template.Spec = *pss
234+
} else {
235+
pss, err := eventingtls.AddTrustBundleVolumes(GetTrustBundleConfigMapLister(ctx), sb, &ps.Spec.Template.Spec)
236+
if err != nil {
237+
logging.FromContext(ctx).Errorw("Failed to add trust bundle volumes %s/%s: %+v", zap.Error(err))
238+
return
239+
}
240+
ps.Spec.Template.Spec = *pss
222241
}
223-
ps.Spec.Template.Spec = *pss
224242

225243
if sb.Status.OIDCTokenSecretName != nil {
226244
ps.Spec.Template.Spec.Volumes = append(ps.Spec.Template.Spec.Volumes, corev1.Volume{
@@ -328,6 +346,20 @@ func (sb *SinkBinding) Undo(ctx context.Context, ps *duckv1.WithPod) {
328346
}
329347
}
330348

349+
type kubeClientKey struct{}
350+
351+
func WithKubeClient(ctx context.Context, k kubernetes.Interface) context.Context {
352+
return context.WithValue(ctx, kubeClientKey{}, k)
353+
}
354+
355+
func getKubeClient(ctx context.Context) kubernetes.Interface {
356+
k := ctx.Value(kubeClientKey{})
357+
if k == nil {
358+
panic("No Kube client found in context.")
359+
}
360+
return k.(kubernetes.Interface)
361+
}
362+
331363
type configMapListerKey struct{}
332364

333365
func WithTrustBundleConfigMapLister(ctx context.Context, lister corev1listers.ConfigMapLister) context.Context {

‎pkg/apis/sources/v1/sinkbinding_lifecycle_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"knative.dev/pkg/apis"
3333
duckv1 "knative.dev/pkg/apis/duck/v1"
3434
"knative.dev/pkg/client/injection/ducks/duck/v1/addressable"
35+
kubeclient "knative.dev/pkg/client/injection/kube/client/fake"
3536
configmapinformer "knative.dev/pkg/client/injection/kube/informers/core/v1/configmap/fake"
3637
fakedynamicclient "knative.dev/pkg/injection/clients/dynamicclient/fake"
3738
"knative.dev/pkg/resolver"
@@ -926,6 +927,7 @@ func TestSinkBindingDo(t *testing.T) {
926927
}
927928
ctx = WithURIResolver(ctx, r)
928929
ctx = WithTrustBundleConfigMapLister(ctx, configmapinformer.Get(ctx).Lister())
930+
ctx = WithKubeClient(ctx, kubeclient.Get(ctx))
929931

930932
for _, cm := range test.configMaps {
931933
_ = configmapinformer.Get(ctx).Informer().GetIndexer().Add(cm)

‎pkg/eventingtls/trust_bundle.go

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -66,16 +66,16 @@ var (
6666

6767
// PropagateTrustBundles propagates Trust bundles ConfigMaps from the system.Namespace() to the
6868
// obj namespace.
69-
func PropagateTrustBundles(ctx context.Context, k8s kubernetes.Interface, trustBundleConfigMapLister corev1listers.ConfigMapLister, gvk schema.GroupVersionKind, obj kmeta.Accessor) error {
69+
func PropagateTrustBundles(ctx context.Context, k8s kubernetes.Interface, trustBundleConfigMapLister corev1listers.ConfigMapLister, gvk schema.GroupVersionKind, obj kmeta.Accessor) ([]*corev1.ConfigMap, error) {
7070

7171
systemNamespaceBundles, err := trustBundleConfigMapLister.ConfigMaps(system.Namespace()).List(TrustBundleSelector)
7272
if err != nil {
73-
return fmt.Errorf("failed to list trust bundle ConfigMaps in %q: %w", system.Namespace(), err)
73+
return nil, fmt.Errorf("failed to list trust bundle ConfigMaps in %q: %w", system.Namespace(), err)
7474
}
7575

7676
userNamespaceBundles, err := trustBundleConfigMapLister.ConfigMaps(obj.GetNamespace()).List(TrustBundleSelector)
7777
if err != nil {
78-
return fmt.Errorf("failed to list trust bundles ConfigMaps in %q: %w", obj.GetNamespace(), err)
78+
return nil, fmt.Errorf("failed to list trust bundles ConfigMaps in %q: %w", obj.GetNamespace(), err)
7979
}
8080

8181
logging.FromContext(ctx).Debugw("Existing bundles",
@@ -85,9 +85,11 @@ func PropagateTrustBundles(ctx context.Context, k8s kubernetes.Interface, trustB
8585

8686
var combinedBundle bytes.Buffer
8787
if err := combineValidTrustBundles(systemNamespaceBundles, &combinedBundle); err != nil {
88-
return err
88+
return nil, err
8989
}
9090

91+
outputUserNamespaceBundles := make([]*corev1.ConfigMap, 0, len(systemNamespaceBundles))
92+
9193
type Pair struct {
9294
sysCM *corev1.ConfigMap
9395
userCm *corev1.ConfigMap
@@ -153,7 +155,7 @@ func PropagateTrustBundles(ctx context.Context, k8s kubernetes.Interface, trustB
153155
// Only delete the ConfigMap if the object owns it
154156
if equality.Semantic.DeepDerivative(expectedOr, or) {
155157
if err := deleteConfigMap(ctx, k8s, obj, p.userCm); err != nil {
156-
return err
158+
return nil, err
157159
}
158160
}
159161
}
@@ -175,8 +177,9 @@ func PropagateTrustBundles(ctx context.Context, k8s kubernetes.Interface, trustB
175177
// Update owner references
176178
expected.OwnerReferences = withOwnerReferences(obj, gvk, []metav1.OwnerReference{})
177179
if err := createConfigMap(ctx, k8s, expected); err != nil {
178-
return err
180+
return nil, err
179181
}
182+
outputUserNamespaceBundles = append(outputUserNamespaceBundles, expected)
180183
continue
181184
}
182185

@@ -185,13 +188,17 @@ func PropagateTrustBundles(ctx context.Context, k8s kubernetes.Interface, trustB
185188
// Update owner references
186189
expected.OwnerReferences = withOwnerReferences(obj, gvk, p.userCm.OwnerReferences)
187190

188-
if !equality.Semantic.DeepDerivative(expected, p.userCm) {
191+
if !equality.Semantic.DeepDerivative(expected.Data, p.userCm.Data) ||
192+
!equality.Semantic.DeepDerivative(expected.BinaryData, p.userCm.BinaryData) ||
193+
!equality.Semantic.DeepDerivative(expected.Labels, p.userCm.Labels) {
189194
if err := updateConfigMap(ctx, k8s, expected); err != nil {
190-
return err
195+
return nil, err
191196
}
192197
}
198+
outputUserNamespaceBundles = append(outputUserNamespaceBundles, expected)
193199
}
194-
return nil
200+
201+
return outputUserNamespaceBundles, nil
195202
}
196203

197204
// CombinedBundlePresent will check if PropagateTrustBundles and AddTrustBundleVolumes will add the TrustBundleCombined
@@ -218,7 +225,10 @@ func AddTrustBundleVolumes(trustBundleLister corev1listers.ConfigMapLister, obj
218225
if err != nil {
219226
return nil, fmt.Errorf("failed to list trust bundles ConfigMaps in %q: %w", obj.GetNamespace(), err)
220227
}
228+
return AddTrustBundleVolumesFromConfigMaps(cms, pt)
229+
}
221230

231+
func AddTrustBundleVolumesFromConfigMaps(cms []*corev1.ConfigMap, pt *corev1.PodSpec) (*corev1.PodSpec, error) {
222232
// Sort ConfigMaps by name to ensure consistent ordering
223233
sort.SliceStable(cms, func(i, j int) bool {
224234
return cms[i].Name < cms[j].Name

‎pkg/reconciler/apiserversource/apiserversource.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -464,5 +464,6 @@ func (r *Reconciler) propagateTrustBundles(ctx context.Context, source *v1.ApiSe
464464
Version: v1.SchemeGroupVersion.Version,
465465
Kind: "ApiServerSource",
466466
}
467-
return eventingtls.PropagateTrustBundles(ctx, r.kubeClientSet, r.trustBundleConfigMapLister, gvk, source)
467+
_, err := eventingtls.PropagateTrustBundles(ctx, r.kubeClientSet, r.trustBundleConfigMapLister, gvk, source)
468+
return err
468469
}

‎pkg/reconciler/sinkbinding/controller.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -150,8 +150,9 @@ func NewController(
150150
trustBundleConfigMapLister: trustBundleConfigMapLister,
151151
}
152152

153+
k8s := kubeclient.Get(ctx)
153154
c.WithContext = func(ctx context.Context, b psbinding.Bindable) (context.Context, error) {
154-
return v1.WithTrustBundleConfigMapLister(v1.WithURIResolver(ctx, sbResolver), trustBundleConfigMapLister), nil
155+
return v1.WithKubeClient(v1.WithTrustBundleConfigMapLister(v1.WithURIResolver(ctx, sbResolver), trustBundleConfigMapLister), k8s), nil
155156
}
156157
c.Tracker = impl.Tracker
157158
c.Factory = &duck.CachedInformerFactory{
@@ -225,9 +226,10 @@ func ListAll(ctx context.Context, handler cache.ResourceEventHandler) psbinding.
225226

226227
func WithContextFactory(ctx context.Context, lister corev1listers.ConfigMapLister, handler func(types.NamespacedName)) psbinding.BindableContext {
227228
r := resolver.NewURIResolverFromTracker(ctx, tracker.New(handler, controller.GetTrackerLease(ctx)))
229+
k := kubeclient.Get(ctx)
228230

229231
return func(ctx context.Context, b psbinding.Bindable) (context.Context, error) {
230-
return v1.WithTrustBundleConfigMapLister(v1.WithURIResolver(ctx, r), lister), nil
232+
return v1.WithKubeClient(v1.WithTrustBundleConfigMapLister(v1.WithURIResolver(ctx, r), lister), k), nil
231233
}
232234
}
233235

‎pkg/reconciler/sinkbinding/sinkbinding.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -240,5 +240,6 @@ func (s *SinkBindingSubResourcesReconciler) propagateTrustBundles(ctx context.Co
240240
Version: v1.SchemeGroupVersion.Version,
241241
Kind: "SinkBinding",
242242
}
243-
return eventingtls.PropagateTrustBundles(ctx, s.kubeclient, s.trustBundleConfigMapLister, gvk, sb)
243+
_, err := eventingtls.PropagateTrustBundles(ctx, s.kubeclient, s.trustBundleConfigMapLister, gvk, sb)
244+
return err
244245
}

‎test/rekt/apiserversource_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,12 +105,13 @@ func TestApiServerSourceDataPlaneTLS(t *testing.T) {
105105
knative.WithLoggingConfig,
106106
knative.WithTracingConfig,
107107
k8s.WithEventListener,
108-
//environment.Managed(t),
108+
environment.Managed(t),
109109
eventshub.WithTLS(t),
110110
)
111111

112112
env.ParallelTest(ctx, t, apiserversourcefeatures.SendsEventsWithTLS())
113113
env.ParallelTest(ctx, t, apiserversourcefeatures.SendsEventsWithTLSTrustBundle())
114+
env.ParallelTest(ctx, t, apiserversourcefeatures.SendsEventsWithTLSWithAdditionalTrustBundle())
114115
}
115116

116117
func TestApiServerSourceDataPlane_EventModes(t *testing.T) {

‎test/rekt/channel_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -382,6 +382,7 @@ func TestInMemoryChannelTLS(t *testing.T) {
382382

383383
env.ParallelTest(ctx, t, channel.SubscriptionTLS())
384384
env.ParallelTest(ctx, t, channel.SubscriptionTLSTrustBundle())
385+
env.ParallelTest(ctx, t, channel.SubscriptionTLSWithAdditionalTrustBundle())
385386
}
386387

387388
func TestChannelImplDispatcherAuthenticatesWithOIDC(t *testing.T) {

‎test/rekt/features/apiserversource/data_plane.go

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
duckv1 "knative.dev/pkg/apis/duck/v1"
2727
"knative.dev/pkg/network"
2828
"knative.dev/reconciler-test/pkg/environment"
29+
"knative.dev/reconciler-test/pkg/knative"
2930

3031
"knative.dev/eventing/pkg/eventingtls/eventingtlstesting"
3132
"knative.dev/eventing/test/rekt/resources/addressable"
@@ -272,6 +273,62 @@ func SendsEventsWithTLSTrustBundle() *feature.Feature {
272273
return f
273274
}
274275

276+
func SendsEventsWithTLSWithAdditionalTrustBundle() *feature.Feature {
277+
src := feature.MakeRandomK8sName("apiserversource")
278+
sink := feature.MakeRandomK8sName("sink")
279+
trustBundle := feature.MakeRandomK8sName("trust-bundle")
280+
281+
f := feature.NewFeatureNamed("Send events to TLS sink - additional trust bundle")
282+
283+
f.Prerequisite("should not run when Istio is enabled", featureflags.IstioDisabled())
284+
285+
f.Setup("install sink", eventshub.Install(sink, eventshub.StartReceiverTLS))
286+
287+
f.Setup("Add trust bundle to system namespace", func(ctx context.Context, t feature.T) {
288+
289+
configmap.Install(trustBundle, knative.KnativeNamespaceFromContext(ctx),
290+
configmap.WithLabels(map[string]string{"networking.knative.dev/trust-bundle": "true"}),
291+
configmap.WithData("ca.crt", *eventshub.GetCaCerts(ctx)),
292+
)(ctx, t)
293+
})
294+
295+
sacmName := feature.MakeRandomK8sName("apiserversource")
296+
f.Requirement("Create Service Account for ApiServerSource with RBAC for v1.Event resources",
297+
setupAccountAndRoleForPods(sacmName))
298+
299+
cfg := []manifest.CfgFn{
300+
apiserversource.WithServiceAccountName(sacmName),
301+
apiserversource.WithEventMode(v1.ResourceMode),
302+
apiserversource.WithResources(v1.APIVersionKindSelector{
303+
APIVersion: "v1",
304+
Kind: "Event",
305+
}),
306+
}
307+
308+
f.Requirement("install ApiServerSource", func(ctx context.Context, t feature.T) {
309+
cfg = append(cfg, apiserversource.WithSink(&duckv1.Destination{
310+
URI: &apis.URL{
311+
Scheme: "https", // Force using https
312+
Host: network.GetServiceHostname(sink, environment.FromContext(ctx).Namespace()),
313+
},
314+
CACerts: nil, // CA certs are in the new trust-bundle
315+
}))
316+
apiserversource.Install(src, cfg...)(ctx, t)
317+
})
318+
f.Requirement("ApiServerSource goes ready", apiserversource.IsReady(src))
319+
320+
f.Stable("ApiServerSource as event source").
321+
Must("delivers events on sink with ref",
322+
eventassert.OnStore(sink).
323+
Match(eventassert.MatchKind(eventshub.EventReceived)).
324+
MatchEvent(test.HasType("dev.knative.apiserver.resource.update")).
325+
AtLeast(1),
326+
).
327+
Must("Set sinkURI to HTTPS endpoint", source.ExpectHTTPSSink(apiserversource.Gvr(), src))
328+
329+
return f
330+
}
331+
275332
// SendsEventsWithEventTypes tests apiserversource to a ready broker.
276333
func SendsEventsWithEventTypes() *feature.Feature {
277334
source := feature.MakeRandomK8sName("source")

0 commit comments

Comments
 (0)