Merge pull request #66 from kcp-dev/go-templates

Add support for Go template expressions in PublishedResources

Author: kcp-ci-bot

deploy/crd/kcp.io/syncagent.kcp.io_publishedresources.yaml

@@ -438,6 +438,8 @@ spec:
                                 description: |-
                                   Reference points to a field inside the main object. This reference is
                                   evaluated on both source and destination sides to find the related object.
+
+                                  Deprecated: Use Go templates instead.
                                 properties:
                                   path:
                                     description: |-
@@ -553,6 +555,8 @@ spec:
                             description: |-
                               Reference points to a field inside the main object. This reference is
                               evaluated on both source and destination sides to find the related object.
+
+                              Deprecated: Use Go templates instead.
                             properties:
                               path:
                                 description: |-
@@ -665,7 +669,9 @@ spec:
                             type: object
                         type: object
                       origin:
-                        description: '"service" or "kcp"'
+                        enum:
+                          - service
+                          - kcp
                         type: string
                     required:
                       - identifier

docs/content/publish-resources/.pages

@@ -1,4 +1,5 @@
 nav:
     - index.md
+    - templating.md
     - api-lifecycle.md
     - technical-details.md

docs/content/publish-resources/index.md

@@ -0,0 +1,89 @@
+# Templating
+
+`PublishedResources` allow to use [Go templates](https://pkg.go.dev/text/template) in a number of
+places. A simple template could look like `{{ .Object.spec.secretName | sha3sum }}`.
+
+## General Usage
+
+Users are encouraged to get familiar with the [Go documentation](https://pkg.go.dev/text/template)
+on templates.
+
+Specifically within the agent, the following rules apply when a template is evaluated:
+
+* All templates must evaluate successfully. Any error will cancel the synchronization process for
+  that object, potentially leaving it in a half-finished overall state.
+* Templates should not output random values, as those can lead to reconcile loops and higher load
+  on the service cluster.
+* Any leading and trailing whitespace will be automatically trimmed from the template's output.
+* All "objects" mentioned in this documentation refer technically to an `unstructured.Unstructured`
+  value's `.Object` field, i.e. the JSON-decoded representation of a Kubernetes object.
+
+## Functions
+
+Templates can make use of all functions provided by [sprig/v3](https://masterminds.github.io/sprig/),
+for example `join` or `b64enc`. The agent then adds the following functions:
+
+* `sha3sum STRING`<br>Returns the hex-encoded SHA3-256 hash (32 characters long).
+* `sha3short STRING [LENGTH=20]`<br>Returns the first `LENGTH` characters of the hex-encoded SHA3-256 hash.
+* <del>`shortHash STRING`</del><br>Returns the first 20 characters of the hex-encoded SHA-1 hash.
+  This function is only available for backwards compatibility when migrating `$variable`-based
+  naming rules to use Go templates. New setups should not use this function, but one of the explicitly
+  named ones, like `sha256sum` or `sha3sum`.
+
+## Context
+
+Depending on where a template is used, different data is available inside the template. The following
+is a summary of those different values:
+
+### Primary Object Naming Rules
+
+This is for templates used in `.spec.naming`:
+
+| Name          | Type                  | Description |
+| ------------- | --------------------- | ----------- |
+| `Object`      | `map[string]any`      | the full remote object found in a kcp workspace |
+| `ClusterName` | `logicalcluster.Name` | the internal cluster identifier (e.g. "34hg2j4gh24jdfgf") |
+| `ClusterPath` | `logicalcluster.Path` | the workspace path (e.g. "root:customer:projectx") |
+
+### Related Object Template Source
+
+This is for templates used in `.spec.related[*].object.template` and
+`.spec.related[*].object.namespace.template`:
+
+| Name          | Type                  | Description |
+| ------------- | --------------------- | ----------- |
+| `Side`        | `string`              | set to either one of the possible origin values (`kcp` or `origin`) to indicate for which cluster the template is currently being evaluated for |
+| `Object`      | `map[string]any`      | the primary object belonging to the related object. Since related object templates are evaluated twice (once for the origin side and once for the destination side), object is the primary object on the side the template is evaluated for |
+| `ClusterName` | `logicalcluster.Name` | the internal cluster identifier (e.g. "34hg2j4gh24jdfgf") of the kcp workspace that the synchronization is currently processing; this value is set for both evaluations, regardless of side |
+| `ClusterPath` | `logicalcluster.Path` | the workspace path (e.g. "root:customer:projectx"); this value is set for both evaluations, regardless of side |
+
+These templates are evaluated once on each side of the synchronization.
+
+### Related Object Label Selectors
+
+This is for templates used in `.spec.related[*].object.selector.matchLabels` and
+`.spec.related[*].object.namespace.selector.matchLabels`, both keys and values:
+
+| Name           | Type                  | Description |
+| -------------- | --------------------- | ----------- |
+| `LocalObject`  | `map[string]any`      | the primary object copy on the local side of the sync (i.e. on the service cluster) |
+| `RemoteObject` | `map[string]any`      | the primary object original, in kcp |
+| `ClusterName`  | `logicalcluster.Name` | the internal cluster identifier (e.g. "34hg2j4gh24jdfgf") of the kcp workspace that the synchronization is currently processing (where the remote object exists) |
+| `ClusterPath`  | `logicalcluster.Path` | the workspace path (e.g. "root:customer:projectx") |
+
+If a template for a key evaluates to an empty string, the key-value combination will be omitted from
+the final selector. Empty values however are allowed.
+
+### Related Object Label Selector Rewrites
+
+This is for templates used in `.spec.related[*].object.selector.rewrite.template` and
+`.spec.related[*].object.namespace.selector.rewrite.template`:
+
+| Name            | Type                  | Description |
+| --------------- | --------------------- | ----------- |
+| `Value`         | `string`              | Either the a found namespace name (when a label selector was used to select the source namespaces for related objects) or the name of a found object (when a label selector was used to find objects). In the former case, the template should return the new namespace to use on the destination side, in the latter case it should return the new object name to use on the destination side. |
+| `RelatedObject` | `map[string]any`      | When a rewrite is used to rewrite object names, RelatedObject is the original related object (found on the origin side). This enables you to ignore the given Value entirely and just select anything from the object itself. RelatedObject is `nil` when the rewrite is performed for a namespace. |
+| `LocalObject`   | `map[string]any`      | the primary object copy on the local side of the sync (i.e. on the service cluster) |
+| `RemoteObject`  | `map[string]any`      | the primary object original, in kcp |
+| `ClusterName`   | `logicalcluster.Name` | the internal cluster identifier (e.g. "34hg2j4gh24jdfgf") of the kcp workspace that the synchronization is currently processing (where the remote object exists) |
+| `ClusterPath`   | `logicalcluster.Path` | the workspace path (e.g. "root:customer:projectx") |

docs/content/publish-resources/templating.md

@@ -1,6 +1,6 @@
 module github.com/kcp-dev/api-syncagent
 
-go 1.23.0
+go 1.24.0
 
 replace github.com/kcp-dev/api-syncagent/sdk => ./sdk
 

go.mod

@@ -116,7 +116,7 @@ func ProjectCRD(crd *apiextensionsv1.CustomResourceDefinition, pubRes *syncagent
 func stripUnwantedVersions(crd *apiextensionsv1.CustomResourceDefinition, pubRes *syncagentv1alpha1.PublishedResource) (*apiextensionsv1.CustomResourceDefinition, error) {
 	src := pubRes.Spec.Resource
 
-	//nolint:staticcheck
+	//nolint:staticcheck // .Version is deprecated, but we still support it for now.
 	if src.Version != "" && len(src.Versions) > 0 {
 		return nil, errors.New("cannot configure both .version and .versions in as the source of a PublishedResource")
 	}
@@ -181,7 +181,7 @@ func projectCRDVersions(crd *apiextensionsv1.CustomResourceDefinition, pubRes *s
 
 	// We already validated that Version and Versions can be set at the same time.
 
-	//nolint:staticcheck
+	//nolint:staticcheck // .Version is deprecated, but we still support it for now.
 	if projection.Version != "" {
 		if size := len(crd.Spec.Versions); size != 1 {
 			return nil, fmt.Errorf("cannot project CRD version to a single version %q because it contains %d versions", projection.Version, size)

internal/projection/naming.go

@@ -34,6 +34,7 @@ type Thing struct {
 
 type ThingSpec struct {
 	Username string `json:"username"`
+	Kink     string `json:"kink"`
 	Address  string `json:"address,omitempty"`
 }
 

internal/projection/projection.go

@@ -17,7 +17,6 @@ limitations under the License.
 package sync
 
 import (
-	"context"
 	"testing"
 
 	"github.com/kcp-dev/logicalcluster/v3"
@@ -27,9 +26,9 @@ import (
 
 func TestNewContext(t *testing.T) {
 	clusterName := logicalcluster.Name("foo")
-	ctx := kontext.WithCluster(context.Background(), clusterName)
+	ctx := kontext.WithCluster(t.Context(), clusterName)
 
-	combinedCtx := NewContext(context.Background(), ctx)
+	combinedCtx := NewContext(t.Context(), ctx)
 
 	if combinedCtx.clusterName != clusterName {
 		t.Fatalf("Expected function to recognize the cluster name in the context, but got %q", combinedCtx.clusterName)

internal/sync/apis/dummy/v1alpha1/thing.go

@@ -40,9 +40,12 @@ spec:
               properties:
                 address:
                   type: string
+                kink:
+                  type: string
                 username:
                   type: string
               required:
+                - kink
                 - username
               type: object
           required:

internal/sync/context_test.go

@@ -40,9 +40,12 @@ spec:
               properties:
                 address:
                   type: string
+                kink:
+                  type: string
                 username:
                   type: string
               required:
+                - kink
                 - username
               type: object
           required:

internal/sync/crd/dummy.example.com_namespacedthings.yaml

@@ -40,9 +40,12 @@ spec:
               properties:
                 address:
                   type: string
+                kink:
+                  type: string
                 username:
                   type: string
               required:
+                - kink
                 - username
               type: object
             status:

internal/sync/crd/dummy.example.com_things.yaml

@@ -40,9 +40,12 @@ spec:
               properties:
                 address:
                   type: string
+                kink:
+                  type: string
                 username:
                   type: string
               required:
+                - kink
                 - username
               type: object
             status:

internal/sync/crd/dummy.example.com_thingwithstatuses.yaml

@@ -21,6 +21,7 @@ import (
 
 	dummyv1alpha1 "github.com/kcp-dev/api-syncagent/internal/sync/apis/dummy/v1alpha1"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 )
@@ -32,6 +33,9 @@ func init() {
 	if err := dummyv1alpha1.AddToScheme(testScheme); err != nil {
 		panic(err)
 	}
+	if err := corev1.AddToScheme(testScheme); err != nil {
+		panic(err)
+	}
 }
 
 var nonEmptyTime = metav1.Time{

internal/sync/crd/dummy.example.com_thingwithstatussubresources.yaml

@@ -35,7 +35,7 @@ import (
 	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-type objectCreatorFunc func(source *unstructured.Unstructured) *unstructured.Unstructured
+type objectCreatorFunc func(source *unstructured.Unstructured) (*unstructured.Unstructured, error)
 
 type objectSyncer struct {
 	// When set, the syncer will create a label on the destination object that contains
@@ -134,7 +134,11 @@ func (s *objectSyncer) applyMutations(source, dest syncSide) (syncSide, syncSide
 	// the mutated names available.
 	destObject := dest.object
 	if destObject == nil {
-		destObject = s.destCreator(source.object)
+		var err error
+		destObject, err = s.destCreator(source.object)
+		if err != nil {
+			return source, dest, fmt.Errorf("failed to create destination object: %w", err)
+		}
 	}
 
 	sourceObj, err := s.mutator.MutateSpec(source.object.DeepCopy(), destObject)
@@ -287,7 +291,10 @@ func (s *objectSyncer) syncObjectStatus(log *zap.SugaredLogger, source, dest syn
 
 func (s *objectSyncer) ensureDestinationObject(log *zap.SugaredLogger, source, dest syncSide) error {
 	// create a copy of the source with GVK projected and renaming rules applied
-	destObj := s.destCreator(source.object)
+	destObj, err := s.destCreator(source.object)
+	if err != nil {
+		return fmt.Errorf("failed to create destination object: %w", err)
+	}
 
 	// make sure the target namespace on the destination cluster exists
 	if err := s.ensureNamespace(dest.ctx, log, dest.client, destObj.GetNamespace()); err != nil {

internal/sync/init_test.go

@@ -17,7 +17,6 @@ limitations under the License.
 package sync
 
 import (
-	"context"
 	"testing"
 
 	dummyv1alpha1 "github.com/kcp-dev/api-syncagent/internal/sync/apis/dummy/v1alpha1"
@@ -37,7 +36,7 @@ func TestStateStoreBasics(t *testing.T) {
 	}, withKind("RemoteThing"))
 
 	serviceClusterClient := buildFakeClient()
-	ctx := context.Background()
+	ctx := t.Context()
 	stateNamespace := "kcp-system"
 
 	primaryObjectSide := syncSide{