Simplify member indexing

codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SchemaGenerator.java

@@ -162,6 +162,11 @@ private void writeSchemaMembers(PythonWriter writer, Shape shape) {
             for (MemberShape member : shape.members()) {
                 if (!generatedShapes.contains(member.getTarget()) && !Prelude.isPreludeShape(member.getTarget())) {
                     deferredMembers.put(member, index++);
+                    writer.write("""
+                            # This needs to reference a schema that isn't defined yet.
+                            # It will be populated with a non-null value at the end of the file.
+                            $S: None,
+                            """, member.getMemberName());
                     continue;
                 }
 
@@ -183,7 +188,6 @@ private void writeSchemaMembers(PythonWriter writer, Shape shape) {
                 writer.write("""
                         $S: {
                             "target": $T,
-                            "index": $L,
                             ${?hasTraits}
                             "traits": [
                                 ${C|}
@@ -193,7 +197,6 @@ private void writeSchemaMembers(PythonWriter writer, Shape shape) {
                         """,
                         member.getMemberName(),
                         targetSchemaSymbol,
-                        index,
                         writer.consumer(w -> writeTraits(w, traits)));
 
                 index++;

codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java

@@ -190,6 +190,9 @@ private static void writePyproject(
                     [tool.ruff.lint]
                     ignore = ["F841"]
 
+                    [tool.ruff.format]
+                    skip-magic-trailing-comma = true
+
                     [tool.pytest.ini_options]
                     python_classes = ["!Test"]
                     asyncio_mode = "auto"

designs/serialization.md

@@ -121,7 +121,6 @@ EXAMPLE_STRUCTURE_SCHEMA = Schema.collection(
     members={
         "member": {
             "target": INTEGER,
-            "index": 0,
             "traits": [
                 DefaultTrait(0),
             ],

packages/smithy-aws-event-stream/tests/unit/_private/__init__.py

@@ -38,32 +38,29 @@
 SCHEMA_MESSAGE_EVENT = Schema.collection(
     id=ShapeID("smithy.example#MessageEvent"),
     members={
-        "boolHeader": {"index": 0, "target": BOOLEAN, "traits": [EVENT_HEADER_TRAIT]},
-        "byteHeader": {"index": 1, "target": BYTE, "traits": [EVENT_HEADER_TRAIT]},
-        "shortHeader": {"index": 2, "target": SHORT, "traits": [EVENT_HEADER_TRAIT]},
-        "intHeader": {"index": 3, "target": INTEGER, "traits": [EVENT_HEADER_TRAIT]},
-        "longHeader": {"index": 4, "target": LONG, "traits": [EVENT_HEADER_TRAIT]},
-        "blobHeader": {"index": 5, "target": BLOB, "traits": [EVENT_HEADER_TRAIT]},
-        "stringHeader": {"index": 6, "target": STRING, "traits": [EVENT_HEADER_TRAIT]},
+        "boolHeader": {"target": BOOLEAN, "traits": [EVENT_HEADER_TRAIT]},
+        "byteHeader": {"target": BYTE, "traits": [EVENT_HEADER_TRAIT]},
+        "shortHeader": {"target": SHORT, "traits": [EVENT_HEADER_TRAIT]},
+        "intHeader": {"target": INTEGER, "traits": [EVENT_HEADER_TRAIT]},
+        "longHeader": {"target": LONG, "traits": [EVENT_HEADER_TRAIT]},
+        "blobHeader": {"target": BLOB, "traits": [EVENT_HEADER_TRAIT]},
+        "stringHeader": {"target": STRING, "traits": [EVENT_HEADER_TRAIT]},
         "timestampHeader": {
-            "index": 7,
             "target": TIMESTAMP,
             "traits": [EVENT_HEADER_TRAIT],
         },
-        "bodyMember": {"index": 8, "target": STRING},
+        "bodyMember": {"target": STRING},
     },
 )
 
 SCHEMA_PAYLOAD_EVENT = Schema.collection(
     id=ShapeID("smithy.example#PayloadEvent"),
     members={
         "header": {
-            "index": 0,
             "target": STRING,
             "traits": [EVENT_HEADER_TRAIT, REQUIRED_TRAIT],
         },
         "payload": {
-            "index": 1,
             "target": STRING,
             "traits": [EVENT_PAYLOAD_TRAIT, REQUIRED_TRAIT],
         },
@@ -74,12 +71,10 @@
     id=ShapeID("smithy.example#BlobPayloadEvent"),
     members={
         "header": {
-            "index": 0,
             "target": STRING,
             "traits": [EVENT_HEADER_TRAIT, REQUIRED_TRAIT],
         },
         "payload": {
-            "index": 1,
             "target": BLOB,
             "traits": [EVENT_PAYLOAD_TRAIT, REQUIRED_TRAIT],
         },
@@ -88,7 +83,7 @@
 
 SCHEMA_ERROR_EVENT = Schema.collection(
     id=ShapeID("smithy.example#ErrorEvent"),
-    members={"message": {"index": 0, "target": STRING, "traits": [REQUIRED_TRAIT]}},
+    members={"message": {"target": STRING, "traits": [REQUIRED_TRAIT]}},
     traits=[ERROR_TRAIT],
 )
 
@@ -97,21 +92,21 @@
     shape_type=ShapeType.UNION,
     traits=[STREAMING_TRAIT],
     members={
-        "message": {"index": 0, "target": SCHEMA_MESSAGE_EVENT},
-        "payload": {"index": 1, "target": SCHEMA_PAYLOAD_EVENT},
-        "blobPayload": {"index": 2, "target": SCHEMA_BLOB_PAYLOAD_EVENT},
-        "error": {"index": 3, "target": SCHEMA_ERROR_EVENT},
+        "message": {"target": SCHEMA_MESSAGE_EVENT},
+        "payload": {"target": SCHEMA_PAYLOAD_EVENT},
+        "blobPayload": {"target": SCHEMA_BLOB_PAYLOAD_EVENT},
+        "error": {"target": SCHEMA_ERROR_EVENT},
     },
 )
 
 SCHEMA_INITIAL_MESSAGE = Schema.collection(
     id=ShapeID("smithy.example#EventStreamOperationInputOutput"),
     members={
-        "message": {"index": 0, "target": STRING},
+        "message": {"target": STRING},
         # Event stream members will not be part of the operation input / output
         # shape schemas.
         # "stream": {
-        #     "index": 1,
+        #
         #     "target": SCHEMA_EVENT_STREAM
         # },
     },

packages/smithy-core/src/smithy_core/schemas.py

@@ -74,14 +74,13 @@ def __init__(
             traits = {}
         object.__setattr__(self, "traits", traits)
 
-        if members:
-            if isinstance(members, list):
-                m: dict[str, Schema] = {}
-                for member in members:
-                    m[member.expect_member_name()] = member
-                members = m
-        else:
+        if members is None:
             members = {}
+        elif isinstance(members, list):
+            m: dict[str, Schema] = {}
+            for member in sorted(members, key=lambda m: m.expect_member_index()):
+                m[member.expect_member_name()] = member
+            members = m
         object.__setattr__(self, "members", members)
 
         if member_target is not None:
@@ -194,7 +193,7 @@ def collection(
         id: ShapeID,
         shape_type: ShapeType = ShapeType.STRUCTURE,
         traits: list["Trait | DynamicTrait"] | None = None,
-        members: Mapping[str, "MemberSchema"] | None = None,
+        members: Mapping[str, "MemberSchema | None"] | None = None,
     ) -> Self:
         """Create a schema for a collection shape.
 
@@ -205,15 +204,23 @@ def collection(
             properties, map keys/values, and union variants. In contrast to the main
             constructor, this is a dict of member names to a simplified dict containing
             only ``traits`` and a ``target``. Member schemas will be generated from this.
+
+            If the value is None, it MUST be populated later. This is to allow a preservation
+            of modeled order without having to explicitly provide it and therefore generate
+            a ton of boilerplate.
         """
         struct_members: dict[str, Schema] = {}
         if members:
-            for k in members.keys():
+            for i, (k, member) in enumerate(members.items()):
+                if member is None:
+                    struct_members[k] = None  # type: ignore
+                    continue
+
                 struct_members[k] = cls.member(
                     id=id.with_member(k),
-                    target=members[k]["target"],
-                    index=members[k]["index"],
-                    member_traits=members[k].get("traits"),
+                    target=member["target"],
+                    index=i,
+                    member_traits=member.get("traits"),
                 )
 
         result = cls(
@@ -268,7 +275,6 @@ class MemberSchema(TypedDict):
     """
 
     target: Required[Schema]
-    index: Required[int]
     traits: NotRequired[list["Trait | DynamicTrait"]]
 
 

packages/smithy-core/tests/unit/test_documents.py

@@ -402,7 +402,11 @@ def test_wrap_list_passes_schema_to_member_documents() -> None:
     list_schema = Schema.collection(
         id=id,
         shape_type=ShapeType.LIST,
-        members={"member": {"target": target_schema, "index": 0}},
+        members={
+            "member": {
+                "target": target_schema,
+            }
+        },
     )
     document = Document(["foo"], schema=list_schema)
     actual = document[0]._schema  # pyright: ignore[reportPrivateUsage]
@@ -421,7 +425,11 @@ def test_setitem_on_list_passes_schema_to_member_documents() -> None:
     list_schema = Schema.collection(
         id=id,
         shape_type=ShapeType.LIST,
-        members={"member": {"target": target_schema, "index": 0}},
+        members={
+            "member": {
+                "target": target_schema,
+            }
+        },
     )
     document = Document(["foo"], schema=list_schema)
     document[0] = "bar"
@@ -440,7 +448,11 @@ def test_wrap_structure_passes_schema_to_member_documents() -> None:
     target_schema = Schema(id=ShapeID("smithy.api#String"), shape_type=ShapeType.STRING)
     struct_schema = Schema.collection(
         id=id,
-        members={"stringMember": {"target": target_schema, "index": 0}},
+        members={
+            "stringMember": {
+                "target": target_schema,
+            }
+        },
     )
     document = Document({"stringMember": "foo"}, schema=struct_schema)
     actual = document["stringMember"]._schema  # pyright: ignore[reportPrivateUsage]
@@ -458,7 +470,11 @@ def test_setitem_on_structure_passes_schema_to_member_documents() -> None:
     target_schema = Schema(id=ShapeID("smithy.api#String"), shape_type=ShapeType.STRING)
     struct_schema = Schema.collection(
         id=id,
-        members={"stringMember": {"target": target_schema, "index": 0}},
+        members={
+            "stringMember": {
+                "target": target_schema,
+            }
+        },
     )
     document = Document({"stringMember": "foo"}, schema=struct_schema)
     document["stringMember"] = "spam"
@@ -478,8 +494,12 @@ def test_wrap_map_passes_schema_to_member_documents() -> None:
         id=id,
         shape_type=ShapeType.MAP,
         members={
-            "key": {"target": STRING, "index": 0},
-            "value": {"target": STRING, "index": 1},
+            "key": {
+                "target": STRING,
+            },
+            "value": {
+                "target": STRING,
+            },
         },
     )
     document = Document({"spam": "eggs"}, schema=map_schema)
@@ -496,8 +516,12 @@ def test_setitem_on_map_passes_schema_to_member_documents() -> None:
         id=id,
         shape_type=ShapeType.MAP,
         members={
-            "key": {"target": STRING, "index": 0},
-            "value": {"target": STRING, "index": 1},
+            "key": {
+                "target": STRING,
+            },
+            "value": {
+                "target": STRING,
+            },
         },
     )
     document = Document({"spam": "eggs"}, schema=map_schema)
@@ -517,47 +541,87 @@ def test_is_none():
 STRING_LIST_SCHEMA = Schema.collection(
     id=ShapeID("smithy.example#StringList"),
     shape_type=ShapeType.LIST,
-    members={"member": {"target": STRING, "index": 0}},
+    members={
+        "member": {
+            "target": STRING,
+        }
+    },
 )
 STRING_MAP_SCHEMA = Schema.collection(
     id=ShapeID("smithy.example#StringMap"),
     shape_type=ShapeType.MAP,
     members={
-        "key": {"target": STRING, "index": 0},
-        "value": {"target": STRING, "index": 1},
+        "key": {
+            "target": STRING,
+        },
+        "value": {
+            "target": STRING,
+        },
     },
 )
 SPARSE_STRING_LIST_SCHEMA = Schema.collection(
     id=ShapeID("smithy.example#StringList"),
     shape_type=ShapeType.LIST,
-    members={"member": {"target": STRING, "index": 0}},
+    members={
+        "member": {
+            "target": STRING,
+        }
+    },
     traits=[SparseTrait()],
 )
 SPARSE_STRING_MAP_SCHEMA = Schema.collection(
     id=ShapeID("smithy.example#StringMap"),
     shape_type=ShapeType.MAP,
     members={
-        "key": {"target": STRING, "index": 0},
-        "value": {"target": STRING, "index": 1},
+        "key": {
+            "target": STRING,
+        },
+        "value": {
+            "target": STRING,
+        },
     },
     traits=[SparseTrait()],
 )
 SCHEMA: Schema = Schema.collection(
     id=ShapeID("smithy.example#DocumentSerdeShape"),
     members={
-        "booleanMember": {"target": BOOLEAN, "index": 0},
-        "integerMember": {"target": INTEGER, "index": 1},
-        "floatMember": {"target": FLOAT, "index": 2},
-        "bigDecimalMember": {"target": BIG_DECIMAL, "index": 3},
-        "stringMember": {"target": STRING, "index": 4},
-        "blobMember": {"target": BLOB, "index": 5},
-        "timestampMember": {"target": TIMESTAMP, "index": 6},
-        "documentMember": {"target": DOCUMENT, "index": 7},
-        "listMember": {"target": STRING_LIST_SCHEMA, "index": 8},
-        "mapMember": {"target": STRING_MAP_SCHEMA, "index": 9},
-        # Index 10 is set below because it's a self-referential member.
-        "sparseListMember": {"target": SPARSE_STRING_LIST_SCHEMA, "index": 11},
-        "sparseMapMember": {"target": SPARSE_STRING_MAP_SCHEMA, "index": 12},
+        "booleanMember": {
+            "target": BOOLEAN,
+        },
+        "integerMember": {
+            "target": INTEGER,
+        },
+        "floatMember": {
+            "target": FLOAT,
+        },
+        "bigDecimalMember": {
+            "target": BIG_DECIMAL,
+        },
+        "stringMember": {
+            "target": STRING,
+        },
+        "blobMember": {
+            "target": BLOB,
+        },
+        "timestampMember": {
+            "target": TIMESTAMP,
+        },
+        "documentMember": {
+            "target": DOCUMENT,
+        },
+        "listMember": {
+            "target": STRING_LIST_SCHEMA,
+        },
+        "mapMember": {
+            "target": STRING_MAP_SCHEMA,
+        },
+        "structMember": None,
+        "sparseListMember": {
+            "target": SPARSE_STRING_LIST_SCHEMA,
+        },
+        "sparseMapMember": {
+            "target": SPARSE_STRING_MAP_SCHEMA,
+        },
     },
 )
 SCHEMA.members["structMember"] = Schema.member(