Skip to content

Commit 03bdcf5

Browse files
joelanfordjoelanford
committed
add WalkMetasFS and WalkMetasReader functions
These function are like WalkFS except that callers provide a function that handles a Meta object at a time, rather than an entire file's worth of FBC objects. For callers that can operate on a single meta at a time, these functions will provide significant reductions in memory usage. Signed-off-by: Joe Lanford <[email protected]>
1 parent 45f491b commit 03bdcf5

File tree

3 files changed

+148
-30
lines changed

3 files changed

+148
-30
lines changed

‎alpha/declcfg/declcfg.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ type RelatedImage struct {
8484
type Meta struct {
8585
Schema string
8686
Package string
87+
Name string
8788

8889
Blob json.RawMessage
8990
}
@@ -96,6 +97,7 @@ func (m *Meta) UnmarshalJSON(blob []byte) error {
9697
type tmp struct {
9798
Schema string `json:"schema"`
9899
Package string `json:"package,omitempty"`
100+
Name string `json:"name,omitempty"`
99101
Properties []property.Property `json:"properties,omitempty"`
100102
}
101103
var t tmp
@@ -104,6 +106,7 @@ func (m *Meta) UnmarshalJSON(blob []byte) error {
104106
}
105107
m.Schema = t.Schema
106108
m.Package = t.Package
109+
m.Name = t.Name
107110
m.Blob = blob
108111
return nil
109112
}

‎alpha/declcfg/load.go

Lines changed: 78 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,74 @@ const (
2222
indexIgnoreFilename = ".indexignore"
2323
)
2424

25+
type WalkMetasFSFunc func(path string, meta *Meta, err error) error
26+
27+
func WalkMetasFS(root fs.FS, walkFn WalkMetasFSFunc) error {
28+
return walkFiles(root, func(root fs.FS, path string, err error) error {
29+
if err != nil {
30+
return walkFn(path, nil, err)
31+
}
32+
33+
f, err := root.Open(path)
34+
if err != nil {
35+
return walkFn(path, nil, err)
36+
}
37+
defer f.Close()
38+
39+
return WalkMetasReader(f, func(meta *Meta, err error) error {
40+
return walkFn(path, meta, err)
41+
})
42+
})
43+
}
44+
45+
type WalkMetasReaderFunc func(meta *Meta, err error) error
46+
47+
func WalkMetasReader(r io.Reader, walkFn WalkMetasReaderFunc) error {
48+
dec := yaml.NewYAMLOrJSONDecoder(r, 4096)
49+
for {
50+
doc := json.RawMessage{}
51+
if err := dec.Decode(&doc); err != nil {
52+
if errors.Is(err, io.EOF) {
53+
break
54+
}
55+
return walkFn(nil, err)
56+
}
57+
doc = []byte(strings.NewReplacer(`\u003c`, "<", `\u003e`, ">", `\u0026`, "&").Replace(string(doc)))
58+
59+
var in Meta
60+
if err := json.Unmarshal(doc, &in); err != nil {
61+
return walkFn(nil, fmt.Errorf("unmarshal error: %s", resolveUnmarshalErr(doc, err)))
62+
}
63+
64+
if err := walkFn(&in, nil); err != nil {
65+
return err
66+
}
67+
}
68+
return nil
69+
}
70+
2571
type WalkFunc func(path string, cfg *DeclarativeConfig, err error) error
2672

2773
// WalkFS walks root using a gitignore-style filename matcher to skip files
2874
// that match patterns found in .indexignore files found throughout the filesystem.
2975
// It calls walkFn for each declarative config file it finds. If WalkFS encounters
3076
// an error loading or parsing any file, the error will be immediately returned.
3177
func WalkFS(root fs.FS, walkFn WalkFunc) error {
78+
return walkFiles(root, func(root fs.FS, path string, err error) error {
79+
if err != nil {
80+
return walkFn(path, nil, err)
81+
}
82+
83+
cfg, err := LoadFile(root, path)
84+
if err != nil {
85+
return walkFn(path, cfg, err)
86+
}
87+
88+
return walkFn(path, cfg, nil)
89+
})
90+
}
91+
92+
func walkFiles(root fs.FS, fn func(root fs.FS, path string, err error) error) error {
3293
if root == nil {
3394
return fmt.Errorf("no declarative config filesystem provided")
3495
}
@@ -40,20 +101,15 @@ func WalkFS(root fs.FS, walkFn WalkFunc) error {
40101

41102
return fs.WalkDir(root, ".", func(path string, info fs.DirEntry, err error) error {
42103
if err != nil {
43-
return walkFn(path, nil, err)
104+
return fn(root, path, err)
44105
}
45106
// avoid validating a directory, an .indexignore file, or any file that matches
46107
// an ignore pattern outlined in a .indexignore file.
47108
if info.IsDir() || info.Name() == indexIgnoreFilename || matcher.Match(path, false) {
48109
return nil
49110
}
50111

51-
cfg, err := LoadFile(root, path)
52-
if err != nil {
53-
return walkFn(path, cfg, err)
54-
}
55-
56-
return walkFn(path, cfg, err)
112+
return fn(root, path, nil)
57113
})
58114
}
59115

@@ -123,46 +179,38 @@ func extractCSV(objs []string) string {
123179
// Path references will not be de-referenced so callers are responsible for de-referencing if necessary.
124180
func LoadReader(r io.Reader) (*DeclarativeConfig, error) {
125181
cfg := &DeclarativeConfig{}
126-
dec := yaml.NewYAMLOrJSONDecoder(r, 4096)
127-
for {
128-
doc := json.RawMessage{}
129-
if err := dec.Decode(&doc); err != nil {
130-
if errors.Is(err, io.EOF) {
131-
break
132-
}
133-
return nil, err
134-
}
135-
doc = []byte(strings.NewReplacer(`\u003c`, "<", `\u003e`, ">", `\u0026`, "&").Replace(string(doc)))
136182

137-
var in Meta
138-
if err := json.Unmarshal(doc, &in); err != nil {
139-
return nil, fmt.Errorf("unmarshal error: %s", resolveUnmarshalErr(doc, err))
183+
if err := WalkMetasReader(r, func(in *Meta, err error) error {
184+
if err != nil {
185+
return err
140186
}
141-
142187
switch in.Schema {
143188
case SchemaPackage:
144189
var p Package
145-
if err := json.Unmarshal(doc, &p); err != nil {
146-
return nil, fmt.Errorf("parse package: %v", err)
190+
if err := json.Unmarshal(in.Blob, &p); err != nil {
191+
return fmt.Errorf("parse package: %v", err)
147192
}
148193
cfg.Packages = append(cfg.Packages, p)
149194
case SchemaChannel:
150195
var c Channel
151-
if err := json.Unmarshal(doc, &c); err != nil {
152-
return nil, fmt.Errorf("parse channel: %v", err)
196+
if err := json.Unmarshal(in.Blob, &c); err != nil {
197+
return fmt.Errorf("parse channel: %v", err)
153198
}
154199
cfg.Channels = append(cfg.Channels, c)
155200
case SchemaBundle:
156201
var b Bundle
157-
if err := json.Unmarshal(doc, &b); err != nil {
158-
return nil, fmt.Errorf("parse bundle: %v", err)
202+
if err := json.Unmarshal(in.Blob, &b); err != nil {
203+
return fmt.Errorf("parse bundle: %v", err)
159204
}
160205
cfg.Bundles = append(cfg.Bundles, b)
161206
case "":
162-
return nil, fmt.Errorf("object '%s' is missing root schema field", string(doc))
207+
return fmt.Errorf("object '%s' is missing root schema field", string(in.Blob))
163208
default:
164-
cfg.Others = append(cfg.Others, in)
209+
cfg.Others = append(cfg.Others, *in)
165210
}
211+
return nil
212+
}); err != nil {
213+
return nil, err
166214
}
167215
return cfg, nil
168216
}
@@ -213,10 +261,10 @@ func formatUnmarshallErrorString(data []byte, errmsg string, offset int64) strin
213261
var pOffset, origOffset int64
214262
origOffset = 0
215263
for origOffset = 0; origOffset < offset; {
216-
pOffset++
217264
if pString[pOffset] != '\n' && pString[pOffset] != ' ' {
218265
origOffset++
219266
}
267+
pOffset++
220268
}
221269
_, _ = sb.WriteString(pString[:pOffset])
222270
_, _ = sb.WriteString(" <== ")

‎alpha/declcfg/load_test.go

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,73 @@ func TestLoadReader(t *testing.T) {
9292
}
9393
}
9494

95+
func TestWalkMetasFS(t *testing.T) {
96+
type spec struct {
97+
name string
98+
fsys fs.FS
99+
assertion require.ErrorAssertionFunc
100+
expectNumPackages int
101+
expectNumChannels int
102+
expectNumBundles int
103+
expectNumOthers int
104+
}
105+
specs := []spec{
106+
{
107+
name: "Error/NilFS",
108+
fsys: nil,
109+
assertion: require.Error,
110+
},
111+
{
112+
name: "Error/NonExistentDir",
113+
fsys: os.DirFS("non/existent/dir/"),
114+
assertion: require.Error,
115+
},
116+
{
117+
name: "Error/Invalid",
118+
fsys: invalidFS,
119+
assertion: require.Error,
120+
},
121+
{
122+
name: "Success/ValidDir",
123+
fsys: validFS,
124+
assertion: require.NoError,
125+
expectNumPackages: 3,
126+
expectNumChannels: 0,
127+
expectNumBundles: 12,
128+
expectNumOthers: 1,
129+
},
130+
}
131+
132+
for _, s := range specs {
133+
t.Run(s.name, func(t *testing.T) {
134+
numPackages, numChannels, numBundles, numOthers := 0, 0, 0, 0
135+
err := WalkMetasFS(s.fsys, func(path string, meta *Meta, err error) error {
136+
if err != nil {
137+
return err
138+
}
139+
switch meta.Schema {
140+
case SchemaPackage:
141+
numPackages++
142+
case SchemaChannel:
143+
numChannels++
144+
case SchemaBundle:
145+
numBundles++
146+
default:
147+
numOthers++
148+
}
149+
return nil
150+
})
151+
s.assertion(t, err)
152+
if err == nil {
153+
assert.Equal(t, s.expectNumPackages, numPackages, "unexpected package count")
154+
assert.Equal(t, s.expectNumChannels, numChannels, "unexpected channel count")
155+
assert.Equal(t, s.expectNumBundles, numBundles, "unexpected bundle count")
156+
assert.Equal(t, s.expectNumOthers, numOthers, "unexpected others count")
157+
}
158+
})
159+
}
160+
}
161+
95162
func TestLoadFS(t *testing.T) {
96163
type spec struct {
97164
name string

0 commit comments

Comments
 (0)