Merge pull request #11 from sachinkumargit/HYP-8063

Add support for mocking download and upload of cache

Author: abdullah-lt

cache/cache.go

@@ -20,13 +20,13 @@ type Cache interface {
 // Rebuilder is an interface represents a rebuild action.
 type Rebuilder interface {
 	// Rebuild rebuilds cache from the files provided with given paths.
-	Rebuild(srcs []string) error
+	Rebuild(srcs []string, mockDownloadUpload bool) error
 }
 
 // Restorer is an interface represents a restore action.
 type Restorer interface {
 	// Restore restores files from the cache provided with given paths.
-	Restore(srcs []string) error
+	Restore(srcs []string, mockDownloadUpload bool) error
 }
 
 // Flusher is an interface represents a flush action.

cache/rebuilder.go

@@ -35,7 +35,7 @@ func NewRebuilder(logger log.Logger, s storage.Storage, a archive.Archive, g key
 }
 
 // Rebuild rebuilds cache from the files provided with given paths.
-func (r rebuilder) Rebuild(srcs []string) error {
+func (r rebuilder) Rebuild(srcs []string, mockDownloadUpload bool) error {
 	level.Info(r.logger).Log("msg", "rebuilding cache")
 
 	now := time.Now()
@@ -77,7 +77,7 @@ func (r rebuilder) Rebuild(srcs []string) error {
 		go func(dst, src string) {
 			defer wg.Done()
 
-			if err := r.rebuild(src, dst); err != nil {
+			if err := r.rebuild(src, dst, mockDownloadUpload); err != nil {
 				errs.Add(fmt.Errorf("upload from <%s> to <%s>, %w", src, dst, err))
 			}
 		}(dst, src)
@@ -95,7 +95,7 @@ func (r rebuilder) Rebuild(srcs []string) error {
 }
 
 // rebuild pushes the archived file to the cache.
-func (r rebuilder) rebuild(src, dst string) error {
+func (r rebuilder) rebuild(src, dst string, mockDownloadUpload bool) error {
 	src, err := filepath.Abs(filepath.Clean(src))
 	if err != nil {
 		return fmt.Errorf("clean source path, %w", err)
@@ -111,6 +111,9 @@ func (r rebuilder) rebuild(src, dst string) error {
 
 		level.Info(r.logger).Log("msg", "archiving directory", "src", src)
 
+		if mockDownloadUpload {
+			return
+		}
 		written, err := r.a.Create([]string{src}, pw)
 		if err != nil {
 			if err := pw.CloseWithError(fmt.Errorf("archive write, pipe writer failed, %w", err)); err != nil {
@@ -126,6 +129,14 @@ func (r rebuilder) rebuild(src, dst string) error {
 	sw := &statWriter{}
 	tr := io.TeeReader(pr, sw)
 
+	if mockDownloadUpload {
+		level.Debug(r.logger).Log(
+			"msg", "archive created",
+			"local", src,
+			"remote", dst,
+		)
+		return nil
+	}
 	if err := r.s.Put(dst, tr); err != nil {
 		err = fmt.Errorf("upload file, pipe reader failed, %w", err)
 		if err := pr.CloseWithError(err); err != nil {

cache/restorer.go

@@ -32,7 +32,7 @@ func NewRestorer(logger log.Logger, s storage.Storage, a archive.Archive, g key.
 }
 
 // Restore restores files from the cache provided with given paths.
-func (r restorer) Restore(dsts []string) error {
+func (r restorer) Restore(dsts []string, mockDownloadUpload bool) error {
 	level.Info(r.logger).Log("msg", "restoring  cache")
 
 	now := time.Now()
@@ -57,7 +57,7 @@ func (r restorer) Restore(dsts []string) error {
 		go func(src, dst string) {
 			defer wg.Done()
 
-			if err := r.restore(src, dst); err != nil {
+			if err := r.restore(src, dst, mockDownloadUpload); err != nil {
 				errs.Add(fmt.Errorf("download from <%s> to <%s>, %w", src, dst, err))
 			}
 		}(src, dst)
@@ -75,7 +75,7 @@ func (r restorer) Restore(dsts []string) error {
 }
 
 // restore fetches the archived file from the cache and restores to the host machine's file system.
-func (r restorer) restore(src, dst string) error {
+func (r restorer) restore(src, dst string, mockDownloadUpload bool) error {
 	var err error
 
 	pr, pw := io.Pipe()
@@ -86,6 +86,9 @@ func (r restorer) restore(src, dst string) error {
 
 		level.Info(r.logger).Log("msg", "downloading archived directory", "remote", src, "local", dst)
 
+		if mockDownloadUpload {
+			return
+		}
 		if err := r.s.Get(src, pw); err != nil {
 			if err := pw.CloseWithError(fmt.Errorf("get file from storage backend, pipe writer failed, %w", err)); err != nil {
 				level.Error(r.logger).Log("msg", "pw close", "err", err)
@@ -95,6 +98,15 @@ func (r restorer) restore(src, dst string) error {
 
 	level.Info(r.logger).Log("msg", "extracting archived directory", "remote", src, "local", dst)
 
+	if mockDownloadUpload {
+		level.Debug(r.logger).Log(
+			"msg", "archive extracted",
+			"local", dst,
+			"remote", src,
+		)
+		return nil
+	}
+
 	written, err := r.a.Extract(dst, pr)
 	if err != nil {
 		err = fmt.Errorf("extract files from downloaded archive, pipe reader failed, %w", err)

internal/plugin/config.go

@@ -33,6 +33,7 @@ type Config struct {
 	Override                bool
 	CompressionLevel        int
 	StorageOperationTimeout time.Duration
+	MockDownloadUpload      bool
 
 	Mount []string
 

internal/plugin/plugin.go

@@ -137,15 +137,15 @@ func (p *Plugin) Exec() error { // nolint: funlen,cyclop
 
 	// 5. Select mode
 	if cfg.Rebuild {
-		if err := c.Rebuild(p.Config.Mount); err != nil {
+		if err := c.Rebuild(p.Config.Mount, cfg.MockDownloadUpload); err != nil {
 			level.Debug(p.logger).Log("err", fmt.Sprintf("%+v\n", err))
 
 			return Error(fmt.Sprintf("[IMPORTANT] build cache, %+v\n", err))
 		}
 	}
 
 	if cfg.Restore {
-		if err := c.Restore(p.Config.Mount); err != nil {
+		if err := c.Restore(p.Config.Mount, cfg.MockDownloadUpload); err != nil {
 			level.Debug(p.logger).Log("err", fmt.Sprintf("%+v\n", err))
 
 			return Error(fmt.Sprintf("[IMPORTANT] restore cache, %+v\n", err))

main.go

@@ -268,6 +268,12 @@ func main() {
 			Value:   true,
 			EnvVars: []string{"PLUGIN_OVERRIDE"},
 		},
+		&cli.BoolFlag{
+			Name:    "mock-download-upload, mkdu",
+			Usage:   "skip downloading and uploading the cache, useful when you want to only display mocked cache logs",
+			EnvVars: []string{"PLUGIN_SKIP_DOWNLOAD"},
+			Value:   false,
+		},
 		// CACHE-KEYS
 		// REBUILD-KEYS
 		// RESTORE-KEYS
@@ -560,6 +566,8 @@ func run(c *cli.Context) error {
 		LocalRoot:        c.String("local-root"),
 		Override:         c.Bool("override"),
 
+		MockDownloadUpload: c.Bool("mock-download-upload"),
+
 		StorageOperationTimeout: c.Duration("backend.operation-timeout"),
 		FileSystem: filesystem.Config{
 			CacheRoot: c.String("filesystem.cache-root"),