Skip to content

Commit 8c5b43d

Browse files
openshift-merge-robotopenshift-merge-robot
authored
Merge pull request #16580 from legionus/image-exclude-from-prune
Automatic merge from submit-queue (batch tested with PRs 16487, 16580). Imagestream tag exclude from pruning Related RFE: https://bugzilla.redhat.com/show_bug.cgi?id=1410018
2 parents 11235db + b70983d commit 8c5b43d

File tree

9 files changed

+163
-21
lines changed

9 files changed

+163
-21
lines changed

‎contrib/completions/bash/oadm

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4746,6 +4746,10 @@ _oadm_prune_images()
47464746
local_nonpersistent_flags+=("--certificate-authority=")
47474747
flags+=("--confirm")
47484748
local_nonpersistent_flags+=("--confirm")
4749+
flags+=("--exclude-imagestreamtag=")
4750+
local_nonpersistent_flags+=("--exclude-imagestreamtag=")
4751+
flags+=("--exclude-imagestreamtag-file=")
4752+
local_nonpersistent_flags+=("--exclude-imagestreamtag-file=")
47494753
flags+=("--force-insecure")
47504754
local_nonpersistent_flags+=("--force-insecure")
47514755
flags+=("--keep-tag-revisions=")

‎contrib/completions/bash/oc

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4911,6 +4911,10 @@ _oc_adm_prune_images()
49114911
local_nonpersistent_flags+=("--certificate-authority=")
49124912
flags+=("--confirm")
49134913
local_nonpersistent_flags+=("--confirm")
4914+
flags+=("--exclude-imagestreamtag=")
4915+
local_nonpersistent_flags+=("--exclude-imagestreamtag=")
4916+
flags+=("--exclude-imagestreamtag-file=")
4917+
local_nonpersistent_flags+=("--exclude-imagestreamtag-file=")
49144918
flags+=("--force-insecure")
49154919
local_nonpersistent_flags+=("--force-insecure")
49164920
flags+=("--keep-tag-revisions=")

‎contrib/completions/bash/openshift

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4746,6 +4746,10 @@ _openshift_admin_prune_images()
47464746
local_nonpersistent_flags+=("--certificate-authority=")
47474747
flags+=("--confirm")
47484748
local_nonpersistent_flags+=("--confirm")
4749+
flags+=("--exclude-imagestreamtag=")
4750+
local_nonpersistent_flags+=("--exclude-imagestreamtag=")
4751+
flags+=("--exclude-imagestreamtag-file=")
4752+
local_nonpersistent_flags+=("--exclude-imagestreamtag-file=")
47494753
flags+=("--force-insecure")
47504754
local_nonpersistent_flags+=("--force-insecure")
47514755
flags+=("--keep-tag-revisions=")
@@ -10076,6 +10080,10 @@ _openshift_cli_adm_prune_images()
1007610080
local_nonpersistent_flags+=("--certificate-authority=")
1007710081
flags+=("--confirm")
1007810082
local_nonpersistent_flags+=("--confirm")
10083+
flags+=("--exclude-imagestreamtag=")
10084+
local_nonpersistent_flags+=("--exclude-imagestreamtag=")
10085+
flags+=("--exclude-imagestreamtag-file=")
10086+
local_nonpersistent_flags+=("--exclude-imagestreamtag-file=")
1007910087
flags+=("--force-insecure")
1008010088
local_nonpersistent_flags+=("--force-insecure")
1008110089
flags+=("--keep-tag-revisions=")

‎contrib/completions/zsh/oadm

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4895,6 +4895,10 @@ _oadm_prune_images()
48954895
local_nonpersistent_flags+=("--certificate-authority=")
48964896
flags+=("--confirm")
48974897
local_nonpersistent_flags+=("--confirm")
4898+
flags+=("--exclude-imagestreamtag=")
4899+
local_nonpersistent_flags+=("--exclude-imagestreamtag=")
4900+
flags+=("--exclude-imagestreamtag-file=")
4901+
local_nonpersistent_flags+=("--exclude-imagestreamtag-file=")
48984902
flags+=("--force-insecure")
48994903
local_nonpersistent_flags+=("--force-insecure")
49004904
flags+=("--keep-tag-revisions=")

‎contrib/completions/zsh/oc

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5060,6 +5060,10 @@ _oc_adm_prune_images()
50605060
local_nonpersistent_flags+=("--certificate-authority=")
50615061
flags+=("--confirm")
50625062
local_nonpersistent_flags+=("--confirm")
5063+
flags+=("--exclude-imagestreamtag=")
5064+
local_nonpersistent_flags+=("--exclude-imagestreamtag=")
5065+
flags+=("--exclude-imagestreamtag-file=")
5066+
local_nonpersistent_flags+=("--exclude-imagestreamtag-file=")
50635067
flags+=("--force-insecure")
50645068
local_nonpersistent_flags+=("--force-insecure")
50655069
flags+=("--keep-tag-revisions=")

‎contrib/completions/zsh/openshift

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4895,6 +4895,10 @@ _openshift_admin_prune_images()
48954895
local_nonpersistent_flags+=("--certificate-authority=")
48964896
flags+=("--confirm")
48974897
local_nonpersistent_flags+=("--confirm")
4898+
flags+=("--exclude-imagestreamtag=")
4899+
local_nonpersistent_flags+=("--exclude-imagestreamtag=")
4900+
flags+=("--exclude-imagestreamtag-file=")
4901+
local_nonpersistent_flags+=("--exclude-imagestreamtag-file=")
48984902
flags+=("--force-insecure")
48994903
local_nonpersistent_flags+=("--force-insecure")
49004904
flags+=("--keep-tag-revisions=")
@@ -10225,6 +10229,10 @@ _openshift_cli_adm_prune_images()
1022510229
local_nonpersistent_flags+=("--certificate-authority=")
1022610230
flags+=("--confirm")
1022710231
local_nonpersistent_flags+=("--confirm")
10232+
flags+=("--exclude-imagestreamtag=")
10233+
local_nonpersistent_flags+=("--exclude-imagestreamtag=")
10234+
flags+=("--exclude-imagestreamtag-file=")
10235+
local_nonpersistent_flags+=("--exclude-imagestreamtag-file=")
1022810236
flags+=("--force-insecure")
1022910237
local_nonpersistent_flags+=("--force-insecure")
1023010238
flags+=("--keep-tag-revisions=")

‎pkg/image/prune/prune.go

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"net/http"
77
"net/url"
88
"reflect"
9+
"regexp"
910
"time"
1011

1112
"github.com/docker/distribution/manifest/schema2"
@@ -59,11 +60,12 @@ const (
5960
// pruneAlgorithm contains the various settings to use when evaluating images
6061
// and layers for pruning.
6162
type pruneAlgorithm struct {
62-
keepYoungerThan time.Time
63-
keepTagRevisions int
64-
pruneOverSizeLimit bool
65-
namespace string
66-
allImages bool
63+
keepYoungerThan time.Time
64+
keepTagRevisions int
65+
pruneOverSizeLimit bool
66+
namespace string
67+
allImages bool
68+
excludeImageStreamTagPatterns []*regexp.Regexp
6769
}
6870

6971
// ImageDeleter knows how to remove images from OpenShift.
@@ -153,6 +155,8 @@ type PrunerOptions struct {
153155
RegistryClient *http.Client
154156
// RegistryURL is the URL of the integrated Docker registry.
155157
RegistryURL *url.URL
158+
// ExcludeImageStreamTagPatterns is the list of regular expressions to exclude an imagesteam tag from pruning.
159+
ExcludeImageStreamTagPatterns []*regexp.Regexp
156160
}
157161

158162
// Pruner knows how to prune istags, images, layers and image configs.
@@ -240,6 +244,7 @@ func NewPruner(options PrunerOptions) (Pruner, kerrors.Aggregate) {
240244
algorithm.allImages = *options.AllImages
241245
}
242246
algorithm.namespace = options.Namespace
247+
algorithm.excludeImageStreamTagPatterns = options.ExcludeImageStreamTagPatterns
243248

244249
p := &pruner{
245250
algorithm: algorithm,
@@ -351,8 +356,30 @@ func (p *pruner) addImageStreamsToGraph(streams *imageapi.ImageStreamList, limit
351356
continue
352357
}
353358

359+
istExcluded := false
360+
dockerImageReference := ""
361+
362+
if p.algorithm.excludeImageStreamTagPatterns != nil {
363+
dockerImageReference = imageapi.DockerImageReference{
364+
Namespace: stream.Namespace,
365+
Name: stream.Name,
366+
Tag: tag,
367+
}.Exact()
368+
369+
for _, pattern := range p.algorithm.excludeImageStreamTagPatterns {
370+
if pattern.MatchString(dockerImageReference) {
371+
istExcluded = true
372+
break
373+
}
374+
}
375+
}
376+
354377
kind := oldImageRevisionReferenceKind
355-
if p.algorithm.pruneOverSizeLimit {
378+
379+
if istExcluded {
380+
glog.V(4).Infof("ImageStreamTag %q is excluded by the regular expression", dockerImageReference)
381+
kind = ReferencedImageEdgeKind
382+
} else if p.algorithm.pruneOverSizeLimit {
356383
if exceedsLimits(stream, imageNode.Image, limits) {
357384
kind = WeakReferencedImageEdgeKind
358385
} else {

‎pkg/oc/admin/prune/images.go

Lines changed: 69 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package prune
22

33
import (
4+
"bufio"
45
"crypto/x509"
56
"encoding/json"
67
"errors"
@@ -86,6 +87,10 @@ var (
8687
# To actually perform the prune operation, the confirm flag must be appended
8788
%[1]s %[2]s --keep-tag-revisions=3 --keep-younger-than=60m --confirm
8889
90+
# To exclude an imagestream from pruning you can specify a regular expression
91+
# which will be applied to the '<namespace>/<name>:<tag>' string.
92+
%[1]s %[2]s --exclude-imagestreamtag='^myproject/hello-openshift:.*$'
93+
8994
# See, what the prune command would delete if we're interested in removing images
9095
# exceeding currently set limit ranges ('openshift.io/Image')
9196
%[1]s %[2]s --prune-over-size-limit
@@ -108,15 +113,17 @@ var (
108113

109114
// PruneImagesOptions holds all the required options for pruning images.
110115
type PruneImagesOptions struct {
111-
Confirm bool
112-
KeepYoungerThan *time.Duration
113-
KeepTagRevisions *int
114-
PruneOverSizeLimit *bool
115-
AllImages *bool
116-
CABundle string
117-
RegistryUrlOverride string
118-
Namespace string
119-
ForceInsecure bool
116+
Confirm bool
117+
KeepYoungerThan *time.Duration
118+
KeepTagRevisions *int
119+
PruneOverSizeLimit *bool
120+
AllImages *bool
121+
CABundle string
122+
RegistryUrlOverride string
123+
Namespace string
124+
ForceInsecure bool
125+
ExcludeImageStreamTag string
126+
ExcludeImageStreamTagFile string
120127

121128
ClientConfig *restclient.Config
122129
AppsClient appsclient.AppsInterface
@@ -158,6 +165,8 @@ func NewCmdPruneImages(f *clientcmd.Factory, parentName, name string, out io.Wri
158165
cmd.Flags().BoolVar(opts.AllImages, "all", *opts.AllImages, "Include images that were imported from external registries as candidates for pruning. If pruned, all the mirrored objects associated with them will also be removed from the integrated registry.")
159166
cmd.Flags().DurationVar(opts.KeepYoungerThan, "keep-younger-than", *opts.KeepYoungerThan, "Specify the minimum age of an image and its referrers for it to be considered a candidate for pruning.")
160167
cmd.Flags().IntVar(opts.KeepTagRevisions, "keep-tag-revisions", *opts.KeepTagRevisions, "Specify the number of image revisions for a tag in an image stream that will be preserved.")
168+
cmd.Flags().StringVar(&opts.ExcludeImageStreamTag, "exclude-imagestreamtag", "", "The regular expression matching ImageStreamTags excluded from pruning.")
169+
cmd.Flags().StringVar(&opts.ExcludeImageStreamTagFile, "exclude-imagestreamtag-file", "", "The filename that contains the regular expressions matching ImageStreamTags excluded from pruning.")
161170
cmd.Flags().BoolVar(opts.PruneOverSizeLimit, "prune-over-size-limit", *opts.PruneOverSizeLimit, "Specify if images which are exceeding LimitRanges (see 'openshift.io/Image'), specified in the same namespace, should be considered for pruning. This flag cannot be combined with --keep-younger-than nor --keep-tag-revisions.")
162171
cmd.Flags().StringVar(&opts.CABundle, "certificate-authority", opts.CABundle, "The path to a certificate authority bundle to use when communicating with the managed Docker registries. Defaults to the certificate authority data from the current user's config file. It cannot be used together with --force-insecure.")
163172
cmd.Flags().StringVar(&opts.RegistryUrlOverride, "registry-url", opts.RegistryUrlOverride, "The address to use when contacting the registry, instead of using the default value. This is useful if you can't resolve or reach the registry (e.g.; the default is a cluster-internal URL) but you do have an alternative route that works. Particular transport protocol can be enforced using '<scheme>://' prefix.")
@@ -387,6 +396,19 @@ func (o PruneImagesOptions) Run() error {
387396
if o.Namespace != metav1.NamespaceAll {
388397
options.Namespace = o.Namespace
389398
}
399+
if len(o.ExcludeImageStreamTagFile) > 0 {
400+
options.ExcludeImageStreamTagPatterns, err = parsePatternsFile(o.ExcludeImageStreamTagFile)
401+
if err != nil {
402+
return err
403+
}
404+
}
405+
if len(o.ExcludeImageStreamTag) > 0 {
406+
pattern, err := regexp.Compile(o.ExcludeImageStreamTag)
407+
if err != nil {
408+
return fmt.Errorf("bad regular expression %q: %v", o.ExcludeImageStreamTag, err)
409+
}
410+
options.ExcludeImageStreamTagPatterns = append(options.ExcludeImageStreamTagPatterns, pattern)
411+
}
390412
pruner, errs := prune.NewPruner(options)
391413
if errs != nil {
392414
o.printGraphBuildErrors(errs)
@@ -770,3 +792,41 @@ func getClientAndMasterVersions(client discovery.DiscoveryInterface, timeout tim
770792

771793
return
772794
}
795+
796+
func parsePatternsFile(filename string) ([]*regexp.Regexp, error) {
797+
file, err := os.Open(filename)
798+
if err != nil {
799+
return nil, err
800+
}
801+
defer file.Close()
802+
803+
var (
804+
patterns []*regexp.Regexp
805+
line string
806+
readerErr error
807+
)
808+
809+
reader := bufio.NewReader(file)
810+
for readerErr == nil {
811+
line, readerErr = reader.ReadString('\n')
812+
813+
if readerErr != nil && readerErr != io.EOF {
814+
return nil, readerErr
815+
}
816+
817+
line = strings.TrimSuffix(line, "\n")
818+
819+
if len(line) == 0 {
820+
continue
821+
}
822+
823+
pattern, err := regexp.Compile(line)
824+
if err != nil {
825+
return nil, fmt.Errorf("%s: bad regular expression %q: %v", filename, line, err)
826+
}
827+
828+
patterns = append(patterns, pattern)
829+
}
830+
831+
return patterns, nil
832+
}

‎test/extended/images/prune.go

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ var _ = g.Describe("[Feature:ImagePrune][registry][Serial] Image prune", func()
103103
}
104104
})
105105

106-
g.It("should prune both internally managed and external images", func() { testPruneAllImages(oc, true, 2) })
106+
g.It("should prune both internally managed and external images", func() { testPruneAllImages(oc, true, 2, false) })
107107
})
108108

109109
g.Describe("with --all=false flag", func() {
@@ -122,7 +122,26 @@ var _ = g.Describe("[Feature:ImagePrune][registry][Serial] Image prune", func()
122122
}
123123
})
124124

125-
g.It("should prune only internally managed images", func() { testPruneAllImages(oc, false, 2) })
125+
g.It("should prune only internally managed images", func() { testPruneAllImages(oc, false, 2, false) })
126+
})
127+
128+
g.Describe("with --all flag, but exclude external image", func() {
129+
g.JustBeforeEach(func() {
130+
if !*originalAcceptSchema2 {
131+
g.By("ensure the registry accepts schema 2")
132+
err := registryutil.EnsureRegistryAcceptsSchema2(oc, true)
133+
o.Expect(err).NotTo(o.HaveOccurred())
134+
}
135+
})
136+
137+
g.AfterEach(func() {
138+
if !*originalAcceptSchema2 {
139+
err := registryutil.EnsureRegistryAcceptsSchema2(oc, false)
140+
o.Expect(err).NotTo(o.HaveOccurred())
141+
}
142+
})
143+
144+
g.It("should prune only internally managed images and ignore external images", func() { testPruneAllImages(oc, false, 2, true) })
126145
})
127146
})
128147

@@ -240,7 +259,7 @@ func testPruneImages(oc *exutil.CLI, schemaVersion int) {
240259
o.Expect(imgPrune.DockerImageMetadata.Size <= keepSize-confirmSize).To(o.BeTrue())
241260
}
242261

243-
func testPruneAllImages(oc *exutil.CLI, setAllImagesToFalse bool, schemaVersion int) {
262+
func testPruneAllImages(oc *exutil.CLI, setAllImagesToFalse bool, schemaVersion int, excludeExternalImage bool) {
244263
isName := fmt.Sprintf("prune-schema%d-all-images-%t", schemaVersion, setAllImagesToFalse)
245264
repository := oc.Namespace() + "/" + isName
246265

@@ -282,14 +301,14 @@ func testPruneAllImages(oc *exutil.CLI, setAllImagesToFalse bool, schemaVersion
282301
o.Expect(inRepository).To(o.Equal(dryRun))
283302
}
284303

285-
if setAllImagesToFalse {
304+
if setAllImagesToFalse || excludeExternalImage {
286305
o.Expect(output).NotTo(o.ContainSubstring(externalImage.Name))
287306
} else {
288307
o.Expect(output).To(o.ContainSubstring(externalImage.Name))
289308
}
290309

291310
for _, layer := range externalImage.DockerImageLayers {
292-
if setAllImagesToFalse {
311+
if setAllImagesToFalse || excludeExternalImage {
293312
o.Expect(output).NotTo(o.ContainSubstring(layer.Name))
294313
} else {
295314
o.Expect(output).To(o.ContainSubstring(layer.Name))
@@ -300,7 +319,7 @@ func testPruneAllImages(oc *exutil.CLI, setAllImagesToFalse bool, schemaVersion
300319
}
301320
globally, inRepository, err := IsBlobStoredInRegistry(oc, digest.Digest(layer.Name), repository)
302321
o.Expect(err).NotTo(o.HaveOccurred())
303-
o.Expect(globally).To(o.Equal(dryRun || setAllImagesToFalse))
322+
o.Expect(globally).To(o.Equal(dryRun || setAllImagesToFalse || excludeExternalImage))
304323
// mirrored blobs are not linked into any repository/_layers directory
305324
o.Expect(inRepository).To(o.BeFalse())
306325
}
@@ -311,6 +330,10 @@ func testPruneAllImages(oc *exutil.CLI, setAllImagesToFalse bool, schemaVersion
311330
args = append(args, "--all=false")
312331
}
313332

333+
if excludeExternalImage {
334+
args = append(args, "--exclude-imagestreamtag=/origin-release:latest")
335+
}
336+
314337
g.By(fmt.Sprintf("dry-running oc adm %s", strings.Join(args, " ")))
315338
output, err := oc.WithoutNamespace().Run("adm").Args(args...).Output()
316339

0 commit comments

Comments
 (0)