Adds an init script and image for running collect_signals without GCP PubSub (#517)

* Add a script and Dockerfile for initing repos for the collector

Signed-off-by: Caleb Brown <[email protected]>

* Add a cloudbuild config for the init image.

Signed-off-by: Caleb Brown <[email protected]>

* Remove unnecessary cat, pin base image in Dockerfile

Signed-off-by: Caleb Brown <[email protected]>

---------

Signed-off-by: Caleb Brown <[email protected]>

Author: calebbrown

.dockerignore

@@ -6,3 +6,5 @@ images
 
 # Ignore Dockerfile - this improve caching.
 **/Dockerfile
+
+!infra/images

infra/cloudbuild/init_collect_signals/cloudbuild.yaml

@@ -0,0 +1,22 @@
+# Copyright 2023 Criticality Score Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['build', '.',
+  '--build-arg', 'COMMIT_SHA=$COMMIT_SHA',
+  '-t', 'gcr.io/openssf/criticality-score-init-collect-signals:$COMMIT_SHA',
+  '-t', 'gcr.io/openssf/criticality-score-init-collect-signals:latest',
+  '-f', 'infra/images/init_collect_signals/Dockerfile']
+images: ['gcr.io/openssf/criticality-score-init-collect-signals']

infra/images/init_collect_signals/Dockerfile

@@ -0,0 +1,22 @@
+# Copyright 2023 Criticality Score Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:slim@sha256:3497ad3a1053bda2c99a766e8764dd27756fdaf84191dd1501405779688abf58
+
+# Add "yq" to the image so the YAML config can be read.
+RUN apt-get update -qqy && apt-get install -qqy yq
+
+WORKDIR /bin
+COPY ./infra/images/init_collect_signals/init.sh ./
+RUN chmod u+x init.sh

infra/images/init_collect_signals/init.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+# Copyright 2023 Criticality Score Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Test usage (from the base dir of the repo):
+# docker build . -f containers/init_collect_signals/Dockerfile -t criticality_score_init_collection
+# docker run
+#    -v /tmp:/output \
+#    -v $HOME/.config/gcloud:/root/.config/gcloud \
+#    -v $HOME/path/to/config.yaml:/etc/config.yaml \
+#    -ti criticality_score_init_collection \
+#    /bin/init.sh /etc/config.yaml
+
+CONFIG_FILE="$1"
+
+# Read the appropriate settings from the YAML config file.
+BUCKET_URL=`yq -r '."additional-params"."input-bucket".url' "$CONFIG_FILE"`
+BUCKET_PREFIX_FILE=`yq -r '."additional-params"."input-bucket"."prefix-file"' "$CONFIG_FILE"`
+OUTPUT_FILE=`yq -r '."additional-params".criticality."local-url-data-file"' "$CONFIG_FILE"`
+echo "bucket url = $BUCKET_URL"
+echo "bucket prefix file = $BUCKET_PREFIX_FILE"
+echo "url data file = $OUTPUT_FILE"
+
+LATEST_PREFIX=`gsutil cat "$BUCKET_URL"/"$BUCKET_PREFIX_FILE"`
+echo "latest prefix = $LATEST_PREFIX"
+
+# Deinfe some temporary files based on OUTPUT_FILE so they're on the same volume.
+TMP_OUTPUT_FILE_1="$OUTPUT_FILE-tmp-1"
+TMP_OUTPUT_FILE_2="$OUTPUT_FILE-tmp-2"
+
+# Iterate through all the files to merge all together.
+touch "$TMP_OUTPUT_FILE_1"
+for file in `gsutil ls "$BUCKET_URL"/"$LATEST_PREFIX"`; do
+    echo "reading $file"
+    # Read the file, remove the header and turn it into a plain list of repos.
+    gsutil cat "$file" | tail -n +2 | cut -d',' -f1 >> "$TMP_OUTPUT_FILE_1"
+done
+
+# Ensure the file contains only one entry per repo, and shuffle it.
+sort "$TMP_OUTPUT_FILE_1" | uniq | shuf > "$TMP_OUTPUT_FILE_2"
+rm "$TMP_OUTPUT_FILE_1"
+
+# Move the final tmp file to the output file to ensure the change is atomic.
+mv "$TMP_OUTPUT_FILE_2" "$OUTPUT_FILE"
+
+echo "wrote $OUTPUT_FILE"