Skip to content

Commit f7ef656

Browse files
rushmash91rushmash91
committed
Add support for custom shapes using simplified schema (aws-controllers-k8s#574)
1 parent 0ab258c commit f7ef656

File tree

11 files changed

+226
-80
lines changed

11 files changed

+226
-80
lines changed

‎go.mod

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
module github.com/aws-controllers-k8s/code-generator
22

3-
go 1.22.0
4-
5-
toolchain go1.22.4
3+
go 1.24.0
64

75
require (
86
github.com/aws-controllers-k8s/pkg v0.0.15
@@ -21,7 +19,7 @@ require (
2119
github.com/samber/lo v1.37.0
2220
github.com/spf13/cobra v1.8.1
2321
github.com/stretchr/testify v1.9.0
24-
golang.org/x/mod v0.17.0
22+
golang.org/x/mod v0.24.0
2523
k8s.io/apimachinery v0.31.0
2624
sigs.k8s.io/controller-runtime v0.19.0
2725
)
@@ -48,17 +46,17 @@ require (
4846
github.com/cloudflare/circl v1.3.7 // indirect
4947
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
5048
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
51-
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
49+
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
5250
github.com/emirpasic/gods v1.18.1 // indirect
5351
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
5452
github.com/fsnotify/fsnotify v1.7.0 // indirect
5553
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
5654
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
5755
github.com/go-git/go-billy/v5 v5.5.0 // indirect
5856
github.com/go-logr/zapr v1.3.0 // indirect
59-
github.com/go-openapi/jsonpointer v0.19.6 // indirect
60-
github.com/go-openapi/jsonreference v0.20.2 // indirect
61-
github.com/go-openapi/swag v0.22.4 // indirect
57+
github.com/go-openapi/jsonpointer v0.21.0 // indirect
58+
github.com/go-openapi/jsonreference v0.21.0 // indirect
59+
github.com/go-openapi/swag v0.23.0 // indirect
6260
github.com/gogo/protobuf v1.3.2 // indirect
6361
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
6462
github.com/golang/protobuf v1.5.4 // indirect
@@ -74,6 +72,7 @@ require (
7472
github.com/josharian/intern v1.0.0 // indirect
7573
github.com/json-iterator/go v1.1.12 // indirect
7674
github.com/kevinburke/ssh_config v1.2.0 // indirect
75+
github.com/kro-run/kro v0.2.2
7776
github.com/mailru/easyjson v0.7.7 // indirect
7877
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
7978
github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -92,27 +91,27 @@ require (
9291
github.com/xanzy/ssh-agent v0.3.3 // indirect
9392
go.uber.org/multierr v1.11.0 // indirect
9493
go.uber.org/zap v1.26.0 // indirect
95-
golang.org/x/crypto v0.31.0 // indirect
96-
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
97-
golang.org/x/net v0.33.0 // indirect
94+
golang.org/x/crypto v0.35.0 // indirect
95+
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
96+
golang.org/x/net v0.36.0 // indirect
9897
golang.org/x/oauth2 v0.21.0 // indirect
99-
golang.org/x/sync v0.10.0 // indirect
100-
golang.org/x/sys v0.28.0 // indirect
101-
golang.org/x/term v0.27.0 // indirect
102-
golang.org/x/text v0.21.0 // indirect
98+
golang.org/x/sync v0.11.0 // indirect
99+
golang.org/x/sys v0.30.0 // indirect
100+
golang.org/x/term v0.29.0 // indirect
101+
golang.org/x/text v0.22.0 // indirect
103102
golang.org/x/time v0.3.0 // indirect
104-
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
103+
golang.org/x/tools v0.24.0 // indirect
105104
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
106105
google.golang.org/protobuf v1.34.2 // indirect
107106
gopkg.in/inf.v0 v0.9.1 // indirect
108107
gopkg.in/warnings.v0 v0.1.2 // indirect
109108
gopkg.in/yaml.v2 v2.4.0 // indirect
110109
gopkg.in/yaml.v3 v3.0.1 // indirect
111110
k8s.io/api v0.31.0 // indirect
112-
k8s.io/apiextensions-apiserver v0.31.0 // indirect
111+
k8s.io/apiextensions-apiserver v0.31.0
113112
k8s.io/client-go v0.31.0 // indirect
114113
k8s.io/klog/v2 v2.130.1 // indirect
115-
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
114+
k8s.io/kube-openapi v0.0.0-20240816214639-573285566f34 // indirect
116115
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
117116
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
118117
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect

‎go.sum

Lines changed: 36 additions & 42 deletions
Large diffs are not rendered by default.

‎pkg/api/load.go

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
2-
31
package api
42

53
import (
@@ -91,7 +89,6 @@ func loadAPI(modelPath, baseImport string, opts ...func(*API)) (*API, error) {
9189
// fmt.Println("\n\n\n\n",*a.Operations["CreateFileSystem"])
9290
// fmt.Println("\n",*a.Shapes["FileSystemDescription"])
9391

94-
9592
if err = a.Setup(); err != nil {
9693
return nil, err
9794
}
@@ -127,11 +124,11 @@ func attachModelFiles(modelPath string, modelFiles ...modelLoader) error {
127124
// pattern passed in. Returns the path of the model file to be loaded. Includes
128125
// all versions of a service model.
129126
//
130-
// e.g:
131-
// models/apis/*/*/api-2.json
127+
// e.g:
128+
// models/apis/*/*/api-2.json
132129
//
133-
// Or with specific model file:
134-
// models/apis/service/version/api-2.json
130+
// Or with specific model file:
131+
// models/apis/service/version/api-2.json
135132
func ExpandModelGlobPath(globs ...string) ([]string, error) {
136133
modelPaths := []string{}
137134

@@ -154,7 +151,7 @@ func ExpandModelGlobPath(globs ...string) ([]string, error) {
154151
// Uses the third from last path element to determine unique service. Only one
155152
// service version will be included.
156153
//
157-
// models/apis/service/version/api-2.json
154+
// models/apis/service/version/api-2.json
158155
func TrimModelServiceVersions(modelPaths []string) (include, exclude []string) {
159156
sort.Strings(modelPaths)
160157

@@ -229,7 +226,7 @@ func (a *API) Setup() error {
229226

230227
a.fixStutterNames()
231228
if err := a.validateShapeNames(); err != nil {
232-
log.Fatalf(err.Error())
229+
log.Fatalf("%v", err.Error())
233230
}
234231
a.renameExportable()
235232
a.applyShapeNameAliases()

‎pkg/config/config.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,22 @@ type Config struct {
5454
// documentdb.
5555
// This will also change the helm chart and image names.
5656
ControllerName string `json:"controller_name,omitempty"`
57+
// CustomShapes defines custom structure types that can be referenced in resource fields.
58+
// The outer map is keyed by shape name (e.g., "ReplicaStatus"), and the inner map
59+
// is keyed by field name with string values representing field types.
60+
//
61+
// Example:
62+
// custom_shapes:
63+
// ReplicaStatus:
64+
// Status: string
65+
// Region: string
66+
// Description: string
67+
// These shapes can be referenced in a resource's fields section using:
68+
// custom_field:
69+
// list_of: ShapeName # For arrays of the shape
70+
// map_of: ShapeName # For maps with the shape as values
71+
//
72+
CustomShapes map[string]map[string]interface{} `json:"custom_shapes,omitempty"`
5773
}
5874

5975
// SDKNames contains information on the SDK Client package. More precisely

‎pkg/model/crd.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -382,8 +382,7 @@ func (r *CRD) GetPrimaryKeyField() (*Field, error) {
382382
var found bool
383383
primaryField, found = r.Fields[fPath]
384384
if !found {
385-
return nil, fmt.Errorf("could not find field with path " + fPath +
386-
" for primary key " + fieldName)
385+
return nil, fmt.Errorf("could not find field with path %s for primary key %s", fPath, fieldName)
387386
}
388387
}
389388
return primaryField, nil
@@ -835,7 +834,7 @@ func (r *CRD) ReferencedServiceNames() (serviceNames []string) {
835834
}
836835
}
837836

838-
for serviceName, _ := range serviceNamesMap {
837+
for serviceName := range serviceNamesMap {
839838
serviceNames = append(serviceNames, serviceName)
840839
}
841840
sort.Strings(serviceNames)

‎pkg/model/model_dynamodb_test.go

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,3 +107,43 @@ func TestDynamoDB_Table(t *testing.T) {
107107
}
108108
assert.Equal(expSpecFieldCamel, attrCamelNames(specFields))
109109
}
110+
111+
func TestDynamoDB_CustomShape_ReplicasState(t *testing.T) {
112+
assert := assert.New(t)
113+
require := require.New(t)
114+
115+
g := testutil.NewModelForServiceWithOptions(t, "dynamodb", &testutil.TestingModelOptions{
116+
GeneratorConfigFile: "generator-with-custom-shapes.yaml",
117+
})
118+
119+
crds, err := g.GetCRDs()
120+
require.Nil(err)
121+
122+
crd := getCRDByName("Table", crds)
123+
require.NotNil(crd)
124+
125+
// Verify the ReplicaStates field exists
126+
assert.Contains(crd.StatusFields, "ReplicaStates")
127+
replicasDescField := crd.StatusFields["ReplicaStates"]
128+
require.NotNil(replicasDescField)
129+
130+
replicasStateShape := replicasDescField.ShapeRef.Shape.MemberRef.Shape
131+
require.NotNil(replicasStateShape)
132+
133+
// Verify all the expected fields exist in the RepicasState shape
134+
expectedFields := []string{
135+
"RegionName",
136+
"RegionStatus",
137+
"RegionStatusDescription",
138+
"RegionStatusPercentProgress",
139+
"RegionInaccessibleDateTime",
140+
}
141+
142+
for _, fieldName := range expectedFields {
143+
assert.Contains(replicasStateShape.MemberRefs, fieldName, "RepicasState shape is missing field: "+fieldName)
144+
field := replicasStateShape.MemberRefs[fieldName]
145+
assert.Equal("string", field.Shape.Type, "Field "+fieldName+" should be of type string")
146+
}
147+
148+
assert.Equal(len(expectedFields), len(replicasStateShape.MemberRefs), "ReplicasState shape has unexpected number of fields")
149+
}

‎pkg/model/multiversion/delta.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,7 @@ func ComputeFieldDeltas(
189189
if !ok2 {
190190
// if a field was renamed and we can't find it in dstNames, something
191191
// very wrong happened during CRD loading.
192-
return nil, fmt.Errorf("cannot find renamed field %s " + newName)
192+
return nil, fmt.Errorf("cannot find renamed field %s in %s", newName, srcName)
193193
}
194194

195195
// mark field as visited, both old and new names.

‎pkg/model/sdk_api.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -134,14 +134,14 @@ func (a *SDKAPI) GetShapeRefFromType(
134134
// TODO(jaypipes): Only handling maps with string keys at the moment...
135135
isMap := strings.HasPrefix(typeOverride, "map[string]")
136136
if isMap {
137-
elemType = typeOverride[11:len(typeOverride)]
137+
elemType = typeOverride[11:]
138138
}
139139
if isSlice {
140-
elemType = typeOverride[2:len(typeOverride)]
140+
elemType = typeOverride[2:]
141141
}
142142
isPtrElem := strings.HasPrefix(elemType, "*")
143143
if isPtrElem {
144-
elemType = elemType[1:len(elemType)]
144+
elemType = elemType[1:]
145145
}
146146
// first check to see if the element type is a scalar and if it is, just
147147
// create a ShapeRef to represent the type.

‎pkg/sdk/custom_shapes.go

Lines changed: 86 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@ import (
1717
"errors"
1818
"fmt"
1919

20-
awssdkmodel "github.com/aws-controllers-k8s/code-generator/pkg/api"
20+
simpleschema "github.com/kro-run/kro/pkg/simpleschema"
21+
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
2122

23+
awssdkmodel "github.com/aws-controllers-k8s/code-generator/pkg/api"
2224
ackmodel "github.com/aws-controllers-k8s/code-generator/pkg/model"
2325
)
2426

@@ -43,6 +45,10 @@ type customShapeInjector struct {
4345
func (h *Helper) InjectCustomShapes(sdkapi *ackmodel.SDKAPI) error {
4446
injector := customShapeInjector{sdkapi}
4547

48+
if err := injector.injectSimpleSchemaShapes(h.cfg.CustomShapes); err != nil {
49+
return err
50+
}
51+
4652
for _, memberShape := range h.cfg.GetCustomMapFieldMembers() {
4753
customShape, err := injector.newMap(memberShape)
4854
if err != nil {
@@ -52,7 +58,6 @@ func (h *Helper) InjectCustomShapes(sdkapi *ackmodel.SDKAPI) error {
5258
sdkapi.API.Shapes[customShape.Shape.ShapeName] = customShape.Shape
5359
sdkapi.CustomShapes = append(sdkapi.CustomShapes, customShape)
5460
}
55-
5661
for _, memberShape := range h.cfg.GetCustomListFieldMembers() {
5762
customShape, err := injector.newList(memberShape)
5863
if err != nil {
@@ -62,10 +67,89 @@ func (h *Helper) InjectCustomShapes(sdkapi *ackmodel.SDKAPI) error {
6267
sdkapi.API.Shapes[customShape.Shape.ShapeName] = customShape.Shape
6368
sdkapi.CustomShapes = append(sdkapi.CustomShapes, customShape)
6469
}
70+
return nil
71+
}
72+
73+
// injectSimpleSchemaShapes processes custom shapes from the top-level custom_shapes
74+
// section and injects them into the SDK API model.
75+
// Only string types are supported - all other types will cause a panic.
76+
func (i *customShapeInjector) injectSimpleSchemaShapes(customShapes map[string]map[string]interface{}) error {
77+
if len(customShapes) == 0 {
78+
return nil
79+
}
80+
81+
apiShapeNames := i.sdkAPI.API.ShapeNames()
82+
for shapeName, fieldsMap := range customShapes {
83+
// check for duplicates
84+
for _, as := range apiShapeNames {
85+
if as == shapeName {
86+
return fmt.Errorf("CustomType name %s already exists in the API", shapeName)
87+
}
88+
}
89+
openAPISchema, err := simpleschema.ToOpenAPISpec(fieldsMap)
90+
if err != nil {
91+
return err
92+
}
93+
94+
// Create and register the base structure shape
95+
shape, shapeRef := i.newStructureShape(shapeName, openAPISchema)
96+
i.sdkAPI.API.Shapes[shape.ShapeName] = shape
97+
i.sdkAPI.CustomShapes = append(i.sdkAPI.CustomShapes, &ackmodel.CustomShape{
98+
Shape: shape,
99+
ShapeRef: shapeRef,
100+
MemberShapeName: nil,
101+
ValueShapeName: nil,
102+
})
103+
}
65104

66105
return nil
67106
}
68107

108+
// newStructureShape creates a base shape with its member fields
109+
func (i *customShapeInjector) newStructureShape(
110+
shapeName string,
111+
openAPISchema *apiextv1.JSONSchemaProps,
112+
) (*awssdkmodel.Shape, *awssdkmodel.ShapeRef) {
113+
shape := &awssdkmodel.Shape{
114+
API: i.sdkAPI.API,
115+
ShapeName: shapeName,
116+
Type: "structure",
117+
Documentation: "// Custom ACK type for " + shapeName,
118+
MemberRefs: make(map[string]*awssdkmodel.ShapeRef),
119+
}
120+
121+
properties := openAPISchema.Properties
122+
for fieldName, propObj := range properties {
123+
propType := propObj.Type
124+
if propType != "string" {
125+
panic(fmt.Sprintf("Field %s in shape %s has non-string type '%s'",
126+
fieldName, shapeName, propType))
127+
}
128+
addStringFieldToShape(i.sdkAPI, shape, fieldName, shapeName)
129+
}
130+
131+
shapeRef := i.createShapeRefForMember(shape)
132+
return shape, shapeRef
133+
}
134+
135+
// addStringFieldToShape adds a string field to the parent shape
136+
func addStringFieldToShape(
137+
sdkapi *ackmodel.SDKAPI,
138+
parentShape *awssdkmodel.Shape,
139+
fieldName string,
140+
shapeName string,
141+
) {
142+
injector := customShapeInjector{sdkapi}
143+
fieldShape := &awssdkmodel.Shape{
144+
API: sdkapi.API,
145+
ShapeName: fieldName,
146+
Type: "string",
147+
}
148+
149+
sdkapi.API.Shapes[fieldShape.ShapeName] = fieldShape
150+
parentShape.MemberRefs[fieldName] = injector.createShapeRefForMember(fieldShape)
151+
}
152+
69153
// createShapeRefForMember creates a minimal ShapeRef type to encapsulate a
70154
// shape.
71155
func (i *customShapeInjector) createShapeRefForMember(shape *awssdkmodel.Shape) *awssdkmodel.ShapeRef {

‎pkg/sdk/helper.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,10 @@ func (h *Helper) API(serviceModelName string) (*model.SDKAPI, error) {
110110
_ = api.ServicePackageDoc()
111111
sdkapi := model.NewSDKAPI(api, h.APIGroupSuffix)
112112

113-
h.InjectCustomShapes(sdkapi)
114-
113+
err := h.InjectCustomShapes(sdkapi)
114+
if err != nil {
115+
return nil, err
116+
}
115117
return sdkapi, nil
116118
}
117119
return nil, ErrServiceNotFound

‎pkg/testdata/models/apis/dynamodb/0000-00-00/generator-with-custom-shapes.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
custom_shapes:
2+
RepicasState:
3+
RegionName: string
4+
RegionStatus: string
5+
RegionStatusDescription: string
6+
RegionStatusPercentProgress: string
7+
RegionInaccessibleDateTime: string
8+
9+
resources:
10+
Table:
11+
fields:
12+
ReplicaStates:
13+
custom_field:
14+
list_of: RepicasState
15+
is_read_only: true

0 commit comments

Comments
 (0)