Giving mirror the option to exit early when --remove is not set

cmd/diff-main.go

@@ -184,7 +184,7 @@ func doDiffMain(ctx context.Context, firstURL, secondURL string) error {
 	}
 
 	// Diff first and second urls.
-	for diffMsg := range objectDifference(ctx, firstClient, secondClient, true) {
+	for diffMsg := range bucketObjectDifference(ctx, firstClient, secondClient) {
 		if diffMsg.Error != nil {
 			errorIf(diffMsg.Error, "Unable to calculate objects difference.")
 			// Ignore error and proceed to next object.

cmd/difference.go

@@ -161,17 +161,23 @@ func metadataEqual(m1, m2 map[string]string) bool {
 	return true
 }
 
-func objectDifference(ctx context.Context, sourceClnt, targetClnt Client, isMetadata bool) (diffCh chan diffMessage) {
+func bucketObjectDifference(ctx context.Context, sourceClnt, targetClnt Client) (diffCh chan diffMessage) {
+	return objectDifference(ctx, sourceClnt, targetClnt, mirrorOptions{
+		isMetadata: false,
+	})
+}
+
+func objectDifference(ctx context.Context, sourceClnt, targetClnt Client, opts mirrorOptions) (diffCh chan diffMessage) {
 	sourceURL := sourceClnt.GetURL().String()
-	sourceCh := sourceClnt.List(ctx, ListOptions{Recursive: true, WithMetadata: isMetadata, ShowDir: DirNone})
+	sourceCh := sourceClnt.List(ctx, ListOptions{Recursive: true, WithMetadata: opts.isMetadata, ShowDir: DirNone})
 
 	targetURL := targetClnt.GetURL().String()
-	targetCh := targetClnt.List(ctx, ListOptions{Recursive: true, WithMetadata: isMetadata, ShowDir: DirNone})
+	targetCh := targetClnt.List(ctx, ListOptions{Recursive: true, WithMetadata: opts.isMetadata, ShowDir: DirNone})
 
-	return difference(sourceURL, sourceCh, targetURL, targetCh, isMetadata, false)
+	return difference(sourceURL, sourceCh, targetURL, targetCh, opts, false)
 }
 
-func bucketDifference(ctx context.Context, sourceClnt, targetClnt Client) (diffCh chan diffMessage) {
+func bucketDifference(ctx context.Context, sourceClnt, targetClnt Client, opts mirrorOptions) (diffCh chan diffMessage) {
 	sourceURL := sourceClnt.GetURL().String()
 	sourceCh := make(chan *ClientContent)
 
@@ -215,11 +221,16 @@ func bucketDifference(ctx context.Context, sourceClnt, targetClnt Client) (diffC
 		}
 	}()
 
-	return difference(sourceURL, sourceCh, targetURL, targetCh, false, false)
+	return difference(sourceURL, sourceCh, targetURL, targetCh, opts, false)
 }
 
-func differenceInternal(sourceURL string, srcCh <-chan *ClientContent, targetURL string, tgtCh <-chan *ClientContent,
-	cmpMetadata, returnSimilar bool, diffCh chan<- diffMessage,
+func differenceInternal(sourceURL string,
+	srcCh <-chan *ClientContent,
+	targetURL string,
+	tgtCh <-chan *ClientContent,
+	opts mirrorOptions,
+	returnSimilar bool,
+	diffCh chan<- diffMessage,
 ) *probe.Error {
 	// Pop first entries from the source and targets
 	srcCtnt, srcOk := <-srcCh
@@ -232,8 +243,14 @@ func differenceInternal(sourceURL string, srcCh <-chan *ClientContent, targetURL
 		tgtEOF = !tgtOk
 
 		// No objects from source AND target: Finish
-		if srcEOF && tgtEOF {
-			break
+		if opts.sourceListingOnly {
+			if srcEOF {
+				break
+			}
+		} else {
+			if srcEOF && tgtEOF {
+				break
+			}
 		}
 
 		if !srcEOF && srcCtnt.Err != nil {
@@ -332,7 +349,7 @@ func differenceInternal(sourceURL string, srcCh <-chan *ClientContent, targetURL
 					firstContent:  srcCtnt,
 					secondContent: tgtCtnt,
 				}
-			} else if cmpMetadata &&
+			} else if opts.isMetadata &&
 				!metadataEqual(srcCtnt.UserMetadata, tgtCtnt.UserMetadata) &&
 				!metadataEqual(srcCtnt.Metadata, tgtCtnt.Metadata) {
 
@@ -375,13 +392,13 @@ func differenceInternal(sourceURL string, srcCh <-chan *ClientContent, targetURL
 
 // objectDifference function finds the difference between all objects
 // recursively in sorted order from source and target.
-func difference(sourceURL string, sourceCh <-chan *ClientContent, targetURL string, targetCh <-chan *ClientContent, cmpMetadata, returnSimilar bool) (diffCh chan diffMessage) {
+func difference(sourceURL string, sourceCh <-chan *ClientContent, targetURL string, targetCh <-chan *ClientContent, opts mirrorOptions, returnSimilar bool) (diffCh chan diffMessage) {
 	diffCh = make(chan diffMessage, 10000)
 
 	go func() {
 		defer close(diffCh)
 
-		err := differenceInternal(sourceURL, sourceCh, targetURL, targetCh, cmpMetadata, returnSimilar, diffCh)
+		err := differenceInternal(sourceURL, sourceCh, targetURL, targetCh, opts, returnSimilar, diffCh)
 		if err != nil {
 			// handle this specifically for filesystem related errors.
 			switch v := err.ToGoError().(type) {

cmd/mirror-main.go

@@ -558,6 +558,13 @@ func (mj *mirrorJob) monitorMirrorStatus(cancel context.CancelFunc) (errDuringMi
 
 	var cancelInProgress bool
 
+	defer func() {
+		// make sure we always cancel the context
+		if !cancelInProgress {
+			cancel()
+		}
+	}()
+
 	for sURLs := range mj.statusCh {
 		if cancelInProgress {
 			// Do not need to print any error after
@@ -1011,6 +1018,13 @@ func runMirror(ctx context.Context, srcURL, dstURL string, cli *cli.Context, enc
 		activeActive:          isWatch,
 	}
 
+	// If we are not using active/active and we are not removing
+	// files from the remote, then we can exit the listing once
+	// local files have been checked for diff.
+	if !mopts.activeActive && !mopts.isRemove {
+		mopts.sourceListingOnly = true
+	}
+
 	// Create a new mirror job and execute it
 	mj := newMirrorJob(srcURL, dstURL, mopts)
 
@@ -1022,7 +1036,7 @@ func runMirror(ctx context.Context, srcURL, dstURL string, cli *cli.Context, enc
 
 	if mirrorSrcBuckets || createDstBuckets {
 		// Synchronize buckets using dirDifference function
-		for d := range bucketDifference(ctx, srcClt, dstClt) {
+		for d := range bucketDifference(ctx, srcClt, dstClt, mj.opts) {
 			if d.Error != nil {
 				if mj.opts.activeActive {
 					errorIf(d.Error, "Failed to start mirroring.. retrying")

cmd/mirror-url.go

@@ -167,7 +167,7 @@ func deltaSourceTarget(ctx context.Context, sourceURL, targetURL string, opts mi
 	}
 
 	// List both source and target, compare and return values through channel.
-	for diffMsg := range objectDifference(ctx, sourceClnt, targetClnt, opts.isMetadata) {
+	for diffMsg := range objectDifference(ctx, sourceClnt, targetClnt, opts) {
 		if diffMsg.Error != nil {
 			// Send all errors through the channel
 			URLsCh <- URLs{Error: diffMsg.Error, ErrorCond: differInUnknown}
@@ -277,6 +277,7 @@ type mirrorOptions struct {
 	storageClass                                          string
 	userMetadata                                          map[string]string
 	checksum                                              minio.ChecksumType
+	sourceListingOnly                                     bool
 }
 
 // Prepares urls that need to be copied or removed based on requested options.