Skip to content

Commit 2180fd6

Browse files
calebbrowncalebbrown
authored
Add support for output enumerated urls in a scorecard compatible format. (#195)
* Add support for output enumerated urls in a scorecard compatible format. Signed-off-by: Caleb Brown <[email protected]> * Fix linter errors Signed-off-by: Caleb Brown <[email protected]> Signed-off-by: Caleb Brown <[email protected]>
1 parent 313bbfc commit 2180fd6

File tree

11 files changed

+280
-21
lines changed

11 files changed

+280
-21
lines changed

‎cmd/enumerate_github/main.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
"go.uber.org/zap/zapcore"
3434

3535
"github.com/ossf/criticality_score/cmd/enumerate_github/githubsearch"
36+
"github.com/ossf/criticality_score/cmd/enumerate_github/repowriter"
3637
"github.com/ossf/criticality_score/internal/envflag"
3738
log "github.com/ossf/criticality_score/internal/log"
3839
"github.com/ossf/criticality_score/internal/outfile"
@@ -62,11 +63,13 @@ var (
6263
endDateFlag = dateFlag(time.Now().UTC().Truncate(oneDay))
6364
logLevel = defaultLogLevel
6465
logEnv log.Env
66+
format repowriter.WriterType
6567

6668
// Maps environment variables to the flags they correspond to.
6769
envFlagMap = envflag.Map{
6870
"CRITICALITY_SCORE_LOG_ENV": "log-env",
6971
"CRITICALITY_SCORE_LOG_LEVEL": "log",
72+
"CRITICALITY_SCORE_FORMAT": "format",
7073
"CRITICALITY_SCORE_WORKERS": "workers",
7174
"CRITICALITY_SCORE_START_DATE": "start",
7275
"CRITICALITY_SCORE_END_DATE": "end",
@@ -103,6 +106,7 @@ func init() {
103106
flag.Var(&startDateFlag, "start", "the start `date` to enumerate back to. Must be at or after 2008-01-01.")
104107
flag.Var(&endDateFlag, "end", "the end `date` to enumerate from.")
105108
flag.Var(&logLevel, "log", "set the `level` of logging.")
109+
textvarflag.TextVar(flag.CommandLine, &format, "format", repowriter.WriterTypeText, "set output file `format`.")
106110
textvarflag.TextVar(flag.CommandLine, &logEnv, "log-env", log.DefaultEnv, "set logging `env`.")
107111
outfile.DefineFlags(flag.CommandLine, "force", "append", "FILE")
108112
flag.Usage = func() {
@@ -205,6 +209,7 @@ func main() {
205209
os.Exit(2)
206210
}
207211
defer out.Close()
212+
w := format.New(out)
208213

209214
logger.With(
210215
zap.String("start", startDateFlag.String()),
@@ -241,7 +246,7 @@ func main() {
241246
totalRepos := 0
242247
go func() {
243248
for repo := range results {
244-
fmt.Fprintln(out, repo)
249+
w.Write(repo)
245250
totalRepos++
246251
}
247252
done <- true

‎cmd/enumerate_github/repowriter/scorecard.go

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package repowriter
2+
3+
import (
4+
"encoding/csv"
5+
"io"
6+
)
7+
8+
var header = []string{"repo", "metadata"}
9+
10+
type scorecardWriter struct {
11+
w *csv.Writer
12+
}
13+
14+
// Scorecard creates a new Writer instance that is used to write a csv file
15+
// of repositories that is compatible with the github.com/ossf/scorecard
16+
// project.
17+
//
18+
// The csv file has a header row with columns "repo" and "metadata". Each
19+
// row consists of the repository url and blank metadata.
20+
func Scorecard(w io.Writer) Writer {
21+
csvWriter := csv.NewWriter(w)
22+
csvWriter.Write(header)
23+
return &scorecardWriter{w: csvWriter}
24+
}
25+
26+
// Write implements the Writer interface.
27+
func (w *scorecardWriter) Write(repo string) error {
28+
if err := w.w.Write([]string{repo, ""}); err != nil {
29+
return err
30+
}
31+
w.w.Flush()
32+
return w.w.Error()
33+
}

‎cmd/enumerate_github/repowriter/scorecard_test.go

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package repowriter_test
2+
3+
import (
4+
"bytes"
5+
"testing"
6+
7+
"github.com/google/go-cmp/cmp"
8+
9+
"github.com/ossf/criticality_score/cmd/enumerate_github/repowriter"
10+
)
11+
12+
func TestScorecardRepoWriter(t *testing.T) {
13+
var buf bytes.Buffer
14+
w := repowriter.Scorecard(&buf)
15+
w.Write("https://github.com/example/example")
16+
w.Write("https://github.com/ossf/criticality_score")
17+
18+
want := "repo,metadata\n" +
19+
"https://github.com/example/example,\n" +
20+
"https://github.com/ossf/criticality_score,\n"
21+
22+
if diff := cmp.Diff(want, buf.String()); diff != "" {
23+
t.Fatalf("Scorecard() mismatch (-want +got):\n%s", diff)
24+
}
25+
}

‎cmd/enumerate_github/repowriter/text.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package repowriter
2+
3+
import (
4+
"fmt"
5+
"io"
6+
)
7+
8+
type textWriter struct {
9+
w io.Writer
10+
}
11+
12+
// Text creates a new Writer instance that is used to write a simple text file
13+
// of repositories, where each line has a single repository url.
14+
func Text(w io.Writer) Writer {
15+
return &textWriter{w}
16+
}
17+
18+
// Write implements the Writer interface.
19+
func (w *textWriter) Write(repo string) error {
20+
_, err := fmt.Fprintln(w.w, repo)
21+
return err
22+
}

‎cmd/enumerate_github/repowriter/text_test.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package repowriter_test
2+
3+
import (
4+
"bytes"
5+
"testing"
6+
7+
"github.com/google/go-cmp/cmp"
8+
9+
"github.com/ossf/criticality_score/cmd/enumerate_github/repowriter"
10+
)
11+
12+
func TestTextRepoWriter(t *testing.T) {
13+
var buf bytes.Buffer
14+
w := repowriter.Text(&buf)
15+
w.Write("https://github.com/example/example")
16+
w.Write("https://github.com/ossf/criticality_score")
17+
18+
want := "https://github.com/example/example\n" +
19+
"https://github.com/ossf/criticality_score\n"
20+
21+
if diff := cmp.Diff(want, buf.String()); diff != "" {
22+
t.Fatalf("Text() mismatch (-want +got):\n%s", diff)
23+
}
24+
}

‎cmd/enumerate_github/repowriter/type.go

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
package repowriter
2+
3+
import (
4+
"bytes"
5+
"errors"
6+
"io"
7+
)
8+
9+
type WriterType int
10+
11+
const (
12+
// WriterTypeText corresponds to the Writer returned by Text.
13+
WriterTypeText = WriterType(iota)
14+
15+
// WriterTypeScorecard corresponds to the Writer returned by Scorecard.
16+
WriterTypeScorecard
17+
)
18+
19+
var ErrorUnknownRepoWriterType = errors.New("unknown repo writer type")
20+
21+
// String implements the fmt.Stringer interface.
22+
func (t WriterType) String() string {
23+
text, err := t.MarshalText()
24+
if err != nil {
25+
return ""
26+
}
27+
return string(text)
28+
}
29+
30+
// MarshalText implements the encoding.TextMarshaler interface.
31+
func (t WriterType) MarshalText() ([]byte, error) {
32+
switch t {
33+
case WriterTypeText:
34+
return []byte("text"), nil
35+
case WriterTypeScorecard:
36+
return []byte("scorecard"), nil
37+
default:
38+
return []byte{}, ErrorUnknownRepoWriterType
39+
}
40+
}
41+
42+
// UnmarshalText implements the encoding.TextUnmarshaler interface.
43+
func (t *WriterType) UnmarshalText(text []byte) error {
44+
switch {
45+
case bytes.Equal(text, []byte("text")):
46+
*t = WriterTypeText
47+
case bytes.Equal(text, []byte("scorecard")):
48+
*t = WriterTypeScorecard
49+
default:
50+
return ErrorUnknownRepoWriterType
51+
}
52+
return nil
53+
}
54+
55+
// New will return a new instance of the corresponding implementation of
56+
// Writer for the given WriterType.
57+
func (t *WriterType) New(w io.Writer) Writer {
58+
switch *t {
59+
case WriterTypeText:
60+
return Text(w)
61+
case WriterTypeScorecard:
62+
return Scorecard(w)
63+
default:
64+
return nil
65+
}
66+
}

‎cmd/enumerate_github/repowriter/type_test.go

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
package repowriter_test
2+
3+
import (
4+
"errors"
5+
"testing"
6+
7+
"github.com/ossf/criticality_score/cmd/enumerate_github/repowriter"
8+
)
9+
10+
func TestTypeString(t *testing.T) {
11+
//nolint:govet
12+
tests := []struct {
13+
name string
14+
writerType repowriter.WriterType
15+
want string
16+
}{
17+
{name: "text", writerType: repowriter.WriterTypeText, want: "text"},
18+
{name: "scorecard", writerType: repowriter.WriterTypeScorecard, want: "scorecard"},
19+
{name: "unknown", writerType: repowriter.WriterType(10), want: ""},
20+
}
21+
for _, test := range tests {
22+
t.Run(test.name, func(t *testing.T) {
23+
got := test.writerType.String()
24+
if got != test.want {
25+
t.Fatalf("String() == %s, want %s", got, test.want)
26+
}
27+
})
28+
}
29+
}
30+
31+
func TestTypeMarshalText(t *testing.T) {
32+
//nolint:govet
33+
tests := []struct {
34+
name string
35+
writerType repowriter.WriterType
36+
want string
37+
err error
38+
}{
39+
{name: "text", writerType: repowriter.WriterTypeText, want: "text"},
40+
{name: "scorecard", writerType: repowriter.WriterTypeScorecard, want: "scorecard"},
41+
{name: "unknown", writerType: repowriter.WriterType(10), want: "", err: repowriter.ErrorUnknownRepoWriterType},
42+
}
43+
for _, test := range tests {
44+
t.Run(test.name, func(t *testing.T) {
45+
got, err := test.writerType.MarshalText()
46+
if err != nil && !errors.Is(err, test.err) {
47+
t.Fatalf("MarhsalText() == %v, want %v", err, test.err)
48+
}
49+
if err == nil {
50+
if test.err != nil {
51+
t.Fatalf("MarshalText() return nil error, want %v", test.err)
52+
}
53+
if string(got) != test.want {
54+
t.Fatalf("MarhsalText() == %s, want %s", got, test.want)
55+
}
56+
}
57+
})
58+
}
59+
}
60+
61+
func TestTypeUnmarshalText(t *testing.T) {
62+
//nolint:govet
63+
tests := []struct {
64+
input string
65+
want repowriter.WriterType
66+
err error
67+
}{
68+
{input: "text", want: repowriter.WriterTypeText},
69+
{input: "scorecard", want: repowriter.WriterTypeScorecard},
70+
{input: "", want: 0, err: repowriter.ErrorUnknownRepoWriterType},
71+
{input: "unknown", want: 0, err: repowriter.ErrorUnknownRepoWriterType},
72+
}
73+
for _, test := range tests {
74+
t.Run(test.input, func(t *testing.T) {
75+
var got repowriter.WriterType
76+
err := got.UnmarshalText([]byte(test.input))
77+
if err != nil && !errors.Is(err, test.err) {
78+
t.Fatalf("UnmarshalText() == %v, want %v", err, test.err)
79+
}
80+
if err == nil {
81+
if test.err != nil {
82+
t.Fatalf("MarshalText() return nil error, want %v", test.err)
83+
}
84+
if got != test.want {
85+
t.Fatalf("UnmarshalText() parsed %d, want %d", int(got), int(test.want))
86+
}
87+
}
88+
})
89+
}
90+
}

‎cmd/enumerate_github/repowriter/writer.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
package repowriter
2+
3+
// Writer is a simple interface for writing a repo. This interface is to
4+
// abstract output formats for lists of repository urls.
5+
type Writer interface {
6+
// Write outputs a single repository url.
7+
Write(repo string) error
8+
}

‎go.mod

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ go 1.18
55
require (
66
cloud.google.com/go/bigquery v1.32.0
77
github.com/blendle/zapdriver v1.3.1
8+
github.com/go-logr/zapr v1.2.3
9+
github.com/google/go-cmp v0.5.8
810
github.com/google/go-github/v44 v44.1.0
911
github.com/iancoleman/strcase v0.2.0
1012
github.com/ossf/scorecard/v4 v4.1.1-0.20220413163106-b00b31646ab4
@@ -42,21 +44,17 @@ require (
4244
github.com/bombsimon/logrusr/v2 v2.0.1 // indirect
4345
github.com/bradleyfalzon/ghinstallation/v2 v2.0.4 // indirect
4446
github.com/go-logr/logr v1.2.3 // indirect
45-
github.com/go-logr/zapr v1.2.3 // indirect
4647
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
4748
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
4849
github.com/golang/protobuf v1.5.2 // indirect
49-
github.com/google/go-cmp v0.5.8 // indirect
5050
github.com/google/go-github/v41 v41.0.0 // indirect
5151
github.com/google/go-querystring v1.1.0 // indirect
5252
github.com/google/uuid v1.3.0 // indirect
5353
github.com/google/wire v0.5.0 // indirect
5454
github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
5555
github.com/googleapis/gax-go/v2 v2.4.0 // indirect
56-
github.com/googleapis/go-type-adapters v1.0.0 // indirect
5756
github.com/jmespath/go-jmespath v0.4.0 // indirect
5857
github.com/shurcooL/graphql v0.0.0-20200928012149-18c5c3165e3a // indirect
59-
github.com/stretchr/testify v1.8.0 // indirect
6058
go.opencensus.io v0.23.0 // indirect
6159
go.uber.org/atomic v1.9.0 // indirect
6260
go.uber.org/multierr v1.8.0 // indirect

0 commit comments

Comments
 (0)