Skip to content

Add support for custom shapes using simplified schema #574

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 20, 2025
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 17 additions & 18 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
module github.com/aws-controllers-k8s/code-generator

go 1.22.0

toolchain go1.22.4
go 1.24.0

require (
github.com/aws-controllers-k8s/pkg v0.0.15
Expand All @@ -21,12 +19,14 @@ require (
github.com/samber/lo v1.37.0
github.com/spf13/cobra v1.8.1
github.com/stretchr/testify v1.9.0
golang.org/x/mod v0.17.0
golang.org/x/mod v0.24.0
k8s.io/apimachinery v0.31.0
sigs.k8s.io/controller-runtime v0.19.0
)

require (
github.com/kro-run/kro v0.2.2
k8s.io/apiextensions-apiserver v0.31.0
dario.cat/mergo v1.0.0 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
Expand All @@ -48,17 +48,17 @@ require (
github.com/cloudflare/circl v1.3.7 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.5.0 // indirect
github.com/go-logr/zapr v1.3.0 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.4 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.21.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
Expand Down Expand Up @@ -92,27 +92,26 @@ require (
github.com/xanzy/ssh-agent v0.3.3 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.26.0 // indirect
golang.org/x/crypto v0.31.0 // indirect
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
golang.org/x/net v0.33.0 // indirect
golang.org/x/crypto v0.35.0 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/net v0.36.0 // indirect
golang.org/x/oauth2 v0.21.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/term v0.27.0 // indirect
golang.org/x/text v0.21.0 // indirect
golang.org/x/sync v0.11.0 // indirect
golang.org/x/sys v0.30.0 // indirect
golang.org/x/term v0.29.0 // indirect
golang.org/x/text v0.22.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
golang.org/x/tools v0.24.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/api v0.31.0 // indirect
k8s.io/apiextensions-apiserver v0.31.0 // indirect
k8s.io/client-go v0.31.0 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
k8s.io/kube-openapi v0.0.0-20240816214639-573285566f34 // indirect
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
Expand Down
78 changes: 36 additions & 42 deletions go.sum

Large diffs are not rendered by default.

15 changes: 6 additions & 9 deletions pkg/api/load.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@


package api

import (
Expand Down Expand Up @@ -91,7 +89,6 @@ func loadAPI(modelPath, baseImport string, opts ...func(*API)) (*API, error) {
// fmt.Println("\n\n\n\n",*a.Operations["CreateFileSystem"])
// fmt.Println("\n",*a.Shapes["FileSystemDescription"])


if err = a.Setup(); err != nil {
return nil, err
}
Expand Down Expand Up @@ -127,11 +124,11 @@ func attachModelFiles(modelPath string, modelFiles ...modelLoader) error {
// pattern passed in. Returns the path of the model file to be loaded. Includes
// all versions of a service model.
//
// e.g:
// models/apis/*/*/api-2.json
// e.g:
// models/apis/*/*/api-2.json
//
// Or with specific model file:
// models/apis/service/version/api-2.json
// Or with specific model file:
// models/apis/service/version/api-2.json
func ExpandModelGlobPath(globs ...string) ([]string, error) {
modelPaths := []string{}

Expand All @@ -154,7 +151,7 @@ func ExpandModelGlobPath(globs ...string) ([]string, error) {
// Uses the third from last path element to determine unique service. Only one
// service version will be included.
//
// models/apis/service/version/api-2.json
// models/apis/service/version/api-2.json
func TrimModelServiceVersions(modelPaths []string) (include, exclude []string) {
sort.Strings(modelPaths)

Expand Down Expand Up @@ -229,7 +226,7 @@ func (a *API) Setup() error {

a.fixStutterNames()
if err := a.validateShapeNames(); err != nil {
log.Fatalf(err.Error())
log.Fatalf("%v", err.Error())
}
a.renameExportable()
a.applyShapeNameAliases()
Expand Down
16 changes: 16 additions & 0 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,22 @@ type Config struct {
// documentdb.
// This will also change the helm chart and image names.
ControllerName string `json:"controller_name,omitempty"`
// CustomShapes defines custom structure types that can be referenced in resource fields.
// The outer map is keyed by shape name (e.g., "ReplicaStatus"), and the inner map
// is keyed by field name with string values representing field types.
//
// Example:
// custom_shapes:
// ReplicaStatus:
// Status: string
// Region: string
// Description: string
// These shapes can be referenced in a resource's fields section using:
// custom_field:
// list_of: ShapeName # For arrays of the shape
// map_of: ShapeName # For maps with the shape as values
//
CustomShapes map[string]map[string]string `json:"custom_shapes,omitempty"`
}

// SDKNames contains information on the SDK Client package. More precisely
Expand Down
3 changes: 1 addition & 2 deletions pkg/model/crd.go
Original file line number Diff line number Diff line change
Expand Up @@ -382,8 +382,7 @@ func (r *CRD) GetPrimaryKeyField() (*Field, error) {
var found bool
primaryField, found = r.Fields[fPath]
if !found {
return nil, fmt.Errorf("could not find field with path " + fPath +
" for primary key " + fieldName)
return nil, fmt.Errorf("could not find field with path %s for primary key %s", fPath, fieldName)
}
}
return primaryField, nil
Expand Down
2 changes: 1 addition & 1 deletion pkg/model/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -412,7 +412,7 @@ func (m *Model) GetTypeDefs() ([]*TypeDef, error) {
if m.typeDefs != nil {
return m.typeDefs, nil
}

tdefs := []*TypeDef{}
// Map, keyed by original Shape GoTypeElem(), with the values being a
// renamed type name (due to conflicting names)
Expand Down
39 changes: 39 additions & 0 deletions pkg/model/model_dynamodb_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,3 +107,42 @@ func TestDynamoDB_Table(t *testing.T) {
}
assert.Equal(expSpecFieldCamel, attrCamelNames(specFields))
}

func TestDynamoDB_CustomShape_ReplicasState(t *testing.T) {
assert := assert.New(t)
require := require.New(t)

g := testutil.NewModelForServiceWithOptions(t, "dynamodb", &testutil.TestingModelOptions{
GeneratorConfigFile: "generator-with-custom-shapes.yaml",
})

crds, err := g.GetCRDs()
require.Nil(err)

crd := getCRDByName("Table", crds)
require.NotNil(crd)

// Verify the ReplicaStates field exists
assert.Contains(crd.StatusFields, "ReplicaStates")
replicasDescField := crd.StatusFields["ReplicaStates"]
require.NotNil(replicasDescField)


replicasStateShape := replicasDescField.ShapeRef.Shape.MemberRef.Shape
require.NotNil(replicasStateShape)

// Verify all the expected fields exist in the RepicasState shape
expectedFields := []string{
"RegionName",
"RegionStatus",
"RegionStatusDescription",
"RegionStatusPercentProgress",
"RegionInaccessibleDateTime",
}

for _, fieldName := range expectedFields {
assert.Contains(replicasStateShape.MemberRefs, fieldName, "RepicasState shape is missing field: "+fieldName)
field := replicasStateShape.MemberRefs[fieldName]
assert.Equal("string", field.Shape.Type, "Field "+fieldName+" should be of type string")
}
}
2 changes: 1 addition & 1 deletion pkg/model/multiversion/delta.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ func ComputeFieldDeltas(
if !ok2 {
// if a field was renamed and we can't find it in dstNames, something
// very wrong happened during CRD loading.
return nil, fmt.Errorf("cannot find renamed field %s " + newName)
return nil, fmt.Errorf("cannot find renamed field %s in %s", newName, srcName)
}

// mark field as visited, both old and new names.
Expand Down
6 changes: 4 additions & 2 deletions pkg/sdk/custom_shapes.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@ type customShapeInjector struct {
// into the list of shapes in the API and update the list of custom shapes in
// the SDKAPI object.
func (h *Helper) InjectCustomShapes(sdkapi *ackmodel.SDKAPI) error {
err := h.InjectSimpleSchemaShapes(sdkapi)
if err != nil {
return err
}
injector := customShapeInjector{sdkapi}

for _, memberShape := range h.cfg.GetCustomMapFieldMembers() {
Expand All @@ -52,7 +56,6 @@ func (h *Helper) InjectCustomShapes(sdkapi *ackmodel.SDKAPI) error {
sdkapi.API.Shapes[customShape.Shape.ShapeName] = customShape.Shape
sdkapi.CustomShapes = append(sdkapi.CustomShapes, customShape)
}

for _, memberShape := range h.cfg.GetCustomListFieldMembers() {
customShape, err := injector.newList(memberShape)
if err != nil {
Expand All @@ -62,7 +65,6 @@ func (h *Helper) InjectCustomShapes(sdkapi *ackmodel.SDKAPI) error {
sdkapi.API.Shapes[customShape.Shape.ShapeName] = customShape.Shape
sdkapi.CustomShapes = append(sdkapi.CustomShapes, customShape)
}

return nil
}

Expand Down
6 changes: 4 additions & 2 deletions pkg/sdk/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,10 @@ func (h *Helper) API(serviceModelName string) (*model.SDKAPI, error) {
_ = api.ServicePackageDoc()
sdkapi := model.NewSDKAPI(api, h.APIGroupSuffix)

h.InjectCustomShapes(sdkapi)

err := h.InjectCustomShapes(sdkapi)
if err != nil {
return nil, err
}
return sdkapi, nil
}
return nil, ErrServiceNotFound
Expand Down
115 changes: 115 additions & 0 deletions pkg/sdk/simpleschema.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
// http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package sdk

import (
"fmt"

awssdkmodel "github.com/aws-controllers-k8s/code-generator/pkg/api"
ackmodel "github.com/aws-controllers-k8s/code-generator/pkg/model"
simpleschema "github.com/kro-run/kro/pkg/simpleschema"
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
)

// InjectSimpleSchemaShapes processes custom shapes from the top-level custom_shapes
// section and injects them into the SDK API model.
// Only string types are supported - all other types will cause a panic.
func (h *Helper) InjectSimpleSchemaShapes(sdkapi *ackmodel.SDKAPI) error {
if len(h.cfg.CustomShapes) == 0 {
return nil
}

apiShapeNames := sdkapi.API.ShapeNames()
for shapeName, fieldsMap := range h.cfg.CustomShapes {
// check dublicates
for _, as := range apiShapeNames {
if as == shapeName {
return fmt.Errorf("shapeName %s already exists in the API", shapeName)
}
}
schemaObj := convertMapValues(fieldsMap)
openAPISchema, err := simpleschema.ToOpenAPISpec(schemaObj)
if err != nil {
return err
}

// Create and register the base structure shape
shape, shapeRef := h.createBaseShape(sdkapi, shapeName, openAPISchema)
sdkapi.API.Shapes[shape.ShapeName] = shape
sdkapi.CustomShapes = append(sdkapi.CustomShapes, &ackmodel.CustomShape{
Shape: shape,
ShapeRef: shapeRef,
MemberShapeName: nil,
ValueShapeName: nil,
})
}

return nil
}

// convertFieldsMapToSchemaObj converts a fields map to a schema object for OpenAPI spec generation
func convertMapValues(fieldsMap map[string]string) map[string]interface{} {
schemaObj := make(map[string]interface{})
for fieldName, fieldType := range fieldsMap {
schemaObj[fieldName] = fieldType
}
return schemaObj
}

// createBaseShape creates a base structure shape with its member fields
func (h *Helper) createBaseShape(
sdkapi *ackmodel.SDKAPI,
shapeName string,
openAPISchema *apiextv1.JSONSchemaProps,
) (*awssdkmodel.Shape, *awssdkmodel.ShapeRef) {
injector := customShapeInjector{sdkapi}
shape := &awssdkmodel.Shape{
API: sdkapi.API,
ShapeName: shapeName,
Type: "structure",
Documentation: "// Custom ACK type for " + shapeName,
MemberRefs: make(map[string]*awssdkmodel.ShapeRef),
}

properties := openAPISchema.Properties
for fieldName, propObj := range properties {
propType := propObj.Type
if propType != "string" {
panic(fmt.Sprintf("Field %s in shape %s has non-string type '%s'",
fieldName, shapeName, propType))
}
addStringFieldToShape(sdkapi, shape, fieldName, shapeName)
}

shapeRef := injector.createShapeRefForMember(shape)
return shape, shapeRef
}

// addStringFieldToShape adds a string field to the parent shape
func addStringFieldToShape(
sdkapi *ackmodel.SDKAPI,
parentShape *awssdkmodel.Shape,
fieldName string,
shapeName string,
) {
injector := customShapeInjector{sdkapi}
fieldShape := &awssdkmodel.Shape{
API: sdkapi.API,
ShapeName: fieldName,
Type: "string",
}

sdkapi.API.Shapes[fieldShape.ShapeName] = fieldShape
parentShape.MemberRefs[fieldName] = injector.createShapeRefForMember(fieldShape)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
custom_shapes:
RepicasState:
RegionName: string
RegionStatus: string
RegionStatusDescription: string
RegionStatusPercentProgress: string
RegionInaccessibleDateTime: string

resources:
Table:
fields:
ReplicaStates:
custom_field:
list_of: RepicasState
is_read_only: true