fix package policy secrets

CHANGELOG.md

@@ -1,5 +1,7 @@
 ## [Unreleased]
 
+- Fix secret handling `elasticstack_fleet_integration_policy` resource. ([#821](https://github.com/elastic/terraform-provider-elasticstack/pull/821))
+
 ## [0.11.8] - 2024-10-02
 
 - Add key_id to the `elasticstack_elasticsearch_api_key` resource. ([#789](https://github.com/elastic/terraform-provider-elasticstack/pull/789))

internal/fleet/integration_policy/create.go

@@ -34,7 +34,7 @@ func (r *integrationPolicyResource) Create(ctx context.Context, req resource.Cre
 		return
 	}
 
-	diags = handleReqRespSecrets(ctx, body, policy, resp.Private)
+	diags = HandleReqRespSecrets(ctx, body, policy, resp.Private)
 	resp.Diagnostics.Append(diags...)
 	if resp.Diagnostics.HasError() {
 		return

internal/fleet/integration_policy/read.go

@@ -34,7 +34,7 @@ func (r *integrationPolicyResource) Read(ctx context.Context, req resource.ReadR
 		return
 	}
 
-	diags = handleRespSecrets(ctx, policy, resp.Private)
+	diags = HandleRespSecrets(ctx, policy, resp.Private)
 	resp.Diagnostics.Append(diags...)
 	if resp.Diagnostics.HasError() {
 		return

internal/fleet/integration_policy/secrets.go

@@ -61,21 +61,30 @@ func (s secretStore) Save(ctx context.Context, private privateData) (diags diag.
 	return private.SetKey(ctx, "secrets", bytes)
 }
 
-// handleRespSecrets extracts the wrapped value from each response var, then
+// HandleRespSecrets extracts the wrapped value from each response var, then
 // replaces any secret refs with the original value from secrets if available.
-func handleRespSecrets(ctx context.Context, resp *fleetapi.PackagePolicy, private privateData) (diags diag.Diagnostics) {
+func HandleRespSecrets(ctx context.Context, resp *fleetapi.PackagePolicy, private privateData) (diags diag.Diagnostics) {
 	secrets, nd := newSecretStore(ctx, resp, private)
 	diags.Append(nd...)
 	if diags.HasError() {
 		return
 	}
 
+	handleVar := func(key string, mval map[string]any, vars map[string]any) {
+		refID := mval["id"].(string)
+		if original, ok := secrets[refID]; ok {
+			vars[key] = original
+		}
+	}
+
 	handleVars := func(vars map[string]any) {
 		for key, val := range vars {
 			if mval, ok := val.(map[string]any); ok {
 				if wrapped, ok := mval["value"]; ok {
 					vars[key] = wrapped
 					val = wrapped
+				} else if v, ok := mval["isSecretRef"]; ok && v == true {
+					handleVar(key, mval, vars)
 				} else {
 					// Don't keep null (missing) values
 					delete(vars, key)
@@ -84,10 +93,7 @@ func handleRespSecrets(ctx context.Context, resp *fleetapi.PackagePolicy, privat
 
 				if mval, ok := val.(map[string]any); ok {
 					if v, ok := mval["isSecretRef"]; ok && v == true {
-						refID := mval["id"].(string)
-						if original, ok := secrets[refID]; ok {
-							vars[key] = original
-						}
+						handleVar(key, mval, vars)
 					}
 				}
 			}
@@ -110,34 +116,49 @@ func handleRespSecrets(ctx context.Context, resp *fleetapi.PackagePolicy, privat
 	return
 }
 
-// handleReqRespSecrets extracts the wrapped value from each response var, then
+// HandleReqRespSecrets extracts the wrapped value from each response var, then
 // maps any secret refs to the original request value.
-func handleReqRespSecrets(ctx context.Context, req fleetapi.PackagePolicyRequest, resp *fleetapi.PackagePolicy, private privateData) (diags diag.Diagnostics) {
+func HandleReqRespSecrets(ctx context.Context, req fleetapi.PackagePolicyRequest, resp *fleetapi.PackagePolicy, private privateData) (diags diag.Diagnostics) {
 	secrets, nd := newSecretStore(ctx, resp, private)
 	diags.Append(nd...)
 	if diags.HasError() {
 		return
 	}
 
+	handleVar := func(key string, mval map[string]any, reqVars map[string]any, respVars map[string]any) {
+		if v, ok := mval["isSecretRef"]; ok && v == true {
+			original := reqVars[key]
+			respVars[key] = original
+
+			// Is the original also a secret ref?
+			// This should only show up during importing and pre 0.11.7 migration.
+			if moriginal, ok := original.(map[string]any); ok {
+				if v, ok := moriginal["isSecretRef"]; ok && v == true {
+					return
+				}
+			}
+
+			refID := mval["id"].(string)
+			secrets[refID] = original
+		}
+	}
+
 	handleVars := func(reqVars map[string]any, respVars map[string]any) {
 		for key, val := range respVars {
 			if mval, ok := val.(map[string]any); ok {
 				if wrapped, ok := mval["value"]; ok {
 					respVars[key] = wrapped
 					val = wrapped
+				} else if v, ok := mval["isSecretRef"]; ok && v == true {
+					handleVar(key, mval, reqVars, respVars)
 				} else {
 					// Don't keep null (missing) values
 					delete(respVars, key)
 					continue
 				}
 
 				if mval, ok := val.(map[string]any); ok {
-					if v, ok := mval["isSecretRef"]; ok && v == true {
-						refID := mval["id"].(string)
-						original := reqVars[key]
-						secrets[refID] = original
-						respVars[key] = original
-					}
+					handleVar(key, mval, reqVars, respVars)
 				}
 			}
 		}

internal/fleet/integration_policy/secrets_test.go

@@ -0,0 +1,252 @@
+package integration_policy_test
+
+import (
+	"context"
+	"maps"
+	"testing"
+
+	fleetapi "github.com/elastic/terraform-provider-elasticstack/generated/fleet"
+	"github.com/elastic/terraform-provider-elasticstack/internal/fleet/integration_policy"
+	"github.com/elastic/terraform-provider-elasticstack/internal/utils"
+	"github.com/hashicorp/terraform-plugin-framework/diag"
+	"github.com/stretchr/testify/require"
+)
+
+type privateData map[string]string
+
+func (p *privateData) GetKey(ctx context.Context, key string) ([]byte, diag.Diagnostics) {
+	if val, ok := (*p)[key]; ok {
+		return []byte(val), nil
+	} else {
+		return nil, nil
+	}
+}
+
+func (p *privateData) SetKey(ctx context.Context, key string, value []byte) diag.Diagnostics {
+	(*p)[key] = string(value)
+	return nil
+}
+
+type Map = map[string]any
+
+func TestHandleRespSecrets(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	private := privateData{"secrets": `{"known-secret":"secret"}`}
+
+	secretRefs := &[]struct {
+		Id *string `json:"id,omitempty"`
+	}{
+		{Id: utils.Pointer("known-secret")},
+	}
+
+	tests := []struct {
+		name  string
+		input Map
+		want  Map
+	}{
+		{
+			name:  "converts plain",
+			input: Map{"k": "v"},
+			want:  Map{"k": "v"},
+		},
+		{
+			name:  "converts wrapped",
+			input: Map{"k": Map{"type": "string", "value": "v"}},
+			want:  Map{"k": "v"},
+		},
+		{
+			name:  "converts wrapped nil",
+			input: Map{"k": Map{"type": "string"}},
+			want:  Map{},
+		},
+		{
+			name:  "converts secret",
+			input: Map{"k": Map{"isSecretRef": true, "id": "known-secret"}},
+			want:  Map{"k": "secret"},
+		},
+		{
+			name:  "converts wrapped secret",
+			input: Map{"k": Map{"type": "password", "value": Map{"isSecretRef": true, "id": "known-secret"}}},
+			want:  Map{"k": "secret"},
+		},
+		{
+			name:  "converts unknown secret",
+			input: Map{"k": Map{"isSecretRef": true, "id": "unknown-secret"}},
+			want:  Map{"k": Map{"isSecretRef": true, "id": "unknown-secret"}},
+		},
+		{
+			name:  "converts wrapped unknown secret",
+			input: Map{"k": Map{"type": "password", "value": Map{"isSecretRef": true, "id": "unknown-secret"}}},
+			want:  Map{"k": Map{"isSecretRef": true, "id": "unknown-secret"}},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			resp := fleetapi.PackagePolicy{
+				SecretReferences: secretRefs,
+				Inputs: map[string]fleetapi.PackagePolicyInput{
+					"input1": {
+						Streams: &Map{"stream1": Map{"vars": maps.Clone(tt.input)}},
+						Vars:    utils.Pointer(maps.Clone(tt.input)),
+					},
+				},
+				Vars: utils.Pointer(maps.Clone(tt.input)),
+			}
+			wants := fleetapi.PackagePolicy{
+				Inputs: map[string]fleetapi.PackagePolicyInput{
+					"input1": {
+						Streams: &Map{"stream1": Map{"vars": tt.want}},
+						Vars:    &tt.want,
+					},
+				},
+				Vars: &tt.want,
+			}
+
+			diags := integration_policy.HandleRespSecrets(ctx, &resp, &private)
+			require.Empty(t, diags)
+			// Policy vars
+			got := *resp.Vars
+			want := *wants.Vars
+			require.Equal(t, want, got)
+
+			// Input vars
+			got = *resp.Inputs["input1"].Vars
+			want = *wants.Inputs["input1"].Vars
+			require.Equal(t, want, got)
+
+			// Stream vars
+			got = (*resp.Inputs["input1"].Streams)["stream1"].(Map)["vars"].(Map)
+			want = (*wants.Inputs["input1"].Streams)["stream1"].(Map)["vars"].(Map)
+			require.Equal(t, want, got)
+
+			// privateData
+			privateWants := privateData{"secrets": `{"known-secret":"secret"}`}
+			require.Equal(t, privateWants, private)
+		})
+	}
+}
+
+func TestHandleReqRespSecrets(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+
+	secretRefs := &[]struct {
+		Id *string `json:"id,omitempty"`
+	}{
+		{Id: utils.Pointer("known-secret")},
+	}
+
+	tests := []struct {
+		name      string
+		reqInput  Map
+		respInput Map
+		want      Map
+	}{
+		{
+			name:      "converts plain",
+			reqInput:  Map{"k": "v"},
+			respInput: Map{"k": "v"},
+			want:      Map{"k": "v"},
+		},
+		{
+			name:      "converts wrapped",
+			reqInput:  Map{"k": "v"},
+			respInput: Map{"k": Map{"type": "string", "value": "v"}},
+			want:      Map{"k": "v"},
+		},
+		{
+			name:      "converts wrapped nil",
+			reqInput:  Map{"k": nil},
+			respInput: Map{"k": Map{"type": "string"}},
+			want:      Map{},
+		},
+		{
+			name:      "converts secret",
+			reqInput:  Map{"k": "secret"},
+			respInput: Map{"k": Map{"isSecretRef": true, "id": "known-secret"}},
+			want:      Map{"k": "secret"},
+		},
+		{
+			name:      "converts wrapped secret",
+			reqInput:  Map{"k": "secret"},
+			respInput: Map{"k": Map{"type": "password", "value": Map{"isSecretRef": true, "id": "known-secret"}}},
+			want:      Map{"k": "secret"},
+		},
+		{
+			name:      "converts unknown secret",
+			reqInput:  Map{"k": Map{"isSecretRef": true, "id": "unknown-secret"}},
+			respInput: Map{"k": Map{"isSecretRef": true, "id": "unknown-secret"}},
+			want:      Map{"k": Map{"isSecretRef": true, "id": "unknown-secret"}},
+		},
+		{
+			name:      "converts wrapped unknown secret",
+			reqInput:  Map{"k": Map{"isSecretRef": true, "id": "unknown-secret"}},
+			respInput: Map{"k": Map{"type": "password", "value": Map{"isSecretRef": true, "id": "unknown-secret"}}},
+			want:      Map{"k": Map{"isSecretRef": true, "id": "unknown-secret"}},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			req := fleetapi.PackagePolicyRequest{
+				Inputs: &map[string]fleetapi.PackagePolicyRequestInput{
+					"input1": {
+						Streams: &map[string]fleetapi.PackagePolicyRequestInputStream{"stream1": {Vars: utils.Pointer(maps.Clone(tt.reqInput))}},
+						Vars:    utils.Pointer(maps.Clone(tt.reqInput)),
+					},
+				},
+				Vars: utils.Pointer(maps.Clone(tt.reqInput)),
+			}
+			resp := fleetapi.PackagePolicy{
+				SecretReferences: secretRefs,
+				Inputs: map[string]fleetapi.PackagePolicyInput{
+					"input1": {
+						Streams: &Map{"stream1": Map{"vars": maps.Clone(tt.respInput)}},
+						Vars:    utils.Pointer(maps.Clone(tt.respInput)),
+					},
+				},
+				Vars: utils.Pointer(maps.Clone(tt.respInput)),
+			}
+			wants := fleetapi.PackagePolicy{
+				Inputs: map[string]fleetapi.PackagePolicyInput{
+					"input1": {
+						Streams: &Map{"stream1": Map{"vars": tt.want}},
+						Vars:    &tt.want,
+					},
+				},
+				Vars: &tt.want,
+			}
+
+			private := privateData{}
+			diags := integration_policy.HandleReqRespSecrets(ctx, req, &resp, &private)
+			require.Empty(t, diags)
+
+			// Policy vars
+			got := *resp.Vars
+			want := *wants.Vars
+			require.Equal(t, want, got)
+
+			// Input vars
+			got = *resp.Inputs["input1"].Vars
+			want = *wants.Inputs["input1"].Vars
+			require.Equal(t, want, got)
+
+			// Stream vars
+			got = (*resp.Inputs["input1"].Streams)["stream1"].(Map)["vars"].(Map)
+			want = (*wants.Inputs["input1"].Streams)["stream1"].(Map)["vars"].(Map)
+			require.Equal(t, want, got)
+
+			if v, ok := (*req.Vars)["k"]; ok && v == "secret" {
+				privateWants := privateData{"secrets": `{"known-secret":"secret"}`}
+				require.Equal(t, privateWants, private)
+			} else {
+				privateWants := privateData{"secrets": `{}`}
+				require.Equal(t, privateWants, private)
+			}
+		})
+	}
+}

internal/fleet/integration_policy/update.go

@@ -35,7 +35,7 @@ func (r *integrationPolicyResource) Update(ctx context.Context, req resource.Upd
 		return
 	}
 
-	diags = handleReqRespSecrets(ctx, body, policy, resp.Private)
+	diags = HandleReqRespSecrets(ctx, body, policy, resp.Private)
 	resp.Diagnostics.Append(diags...)
 	if resp.Diagnostics.HasError() {
 		return