diff --git a/contrib/completions/bash/oc b/contrib/completions/bash/oc index 0b1242a1ba..6671eb92ec 100644 --- a/contrib/completions/bash/oc +++ b/contrib/completions/bash/oc @@ -598,6 +598,10 @@ _oc_adm_catalog_mirror() two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--icsp-scope=") two_word_flags+=("--icsp-scope") local_nonpersistent_flags+=("--icsp-scope") @@ -608,6 +612,8 @@ _oc_adm_catalog_mirror() local_nonpersistent_flags+=("--index-filter-by-os=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--manifests-only") local_nonpersistent_flags+=("--manifests-only") flags+=("--max-components=") @@ -5781,8 +5787,14 @@ _oc_adm_release_extract() two_word_flags+=("--git") local_nonpersistent_flags+=("--git") local_nonpersistent_flags+=("--git=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry") @@ -5893,6 +5905,10 @@ _oc_adm_release_info() two_word_flags+=("--dir") local_nonpersistent_flags+=("--dir") local_nonpersistent_flags+=("--dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--image-for=") two_word_flags+=("--image-for") local_nonpersistent_flags+=("--image-for") @@ -5901,6 +5917,8 @@ _oc_adm_release_info() local_nonpersistent_flags+=("--include-images") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry") @@ -6003,8 +6021,14 @@ _oc_adm_release_mirror() two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry") @@ -6017,6 +6041,10 @@ _oc_adm_release_mirror() local_nonpersistent_flags+=("--registry-config") local_nonpersistent_flags+=("--registry-config=") local_nonpersistent_flags+=("-a") + flags+=("--release-image-icsp-to-dir=") + two_word_flags+=("--release-image-icsp-to-dir") + local_nonpersistent_flags+=("--release-image-icsp-to-dir") + local_nonpersistent_flags+=("--release-image-icsp-to-dir=") flags+=("--release-image-signature-to-dir=") two_word_flags+=("--release-image-signature-to-dir") local_nonpersistent_flags+=("--release-image-signature-to-dir") @@ -6139,12 +6167,18 @@ _oc_adm_release_new() two_word_flags+=("--from-release") local_nonpersistent_flags+=("--from-release") local_nonpersistent_flags+=("--from-release=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--include=") two_word_flags+=("--include") local_nonpersistent_flags+=("--include") local_nonpersistent_flags+=("--include=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--mapping-file=") two_word_flags+=("--mapping-file") local_nonpersistent_flags+=("--mapping-file") @@ -14579,12 +14613,18 @@ _oc_image_append() two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--image=") two_word_flags+=("--image") local_nonpersistent_flags+=("--image") local_nonpersistent_flags+=("--image=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry") @@ -14681,8 +14721,14 @@ _oc_image_extract() two_word_flags+=("--filter-by-os") local_nonpersistent_flags+=("--filter-by-os") local_nonpersistent_flags+=("--filter-by-os=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--only-files") local_nonpersistent_flags+=("--only-files") flags+=("--path=") @@ -14767,8 +14813,14 @@ _oc_image_info() two_word_flags+=("--filter-by-os") local_nonpersistent_flags+=("--filter-by-os") local_nonpersistent_flags+=("--filter-by-os=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--output=") two_word_flags+=("--output") two_word_flags+=("-o") @@ -14865,10 +14917,16 @@ _oc_image_mirror() two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") flags+=("--keep-manifest-list") local_nonpersistent_flags+=("--keep-manifest-list") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry") diff --git a/contrib/completions/zsh/oc b/contrib/completions/zsh/oc index af51260eb1..61e9c58b49 100644 --- a/contrib/completions/zsh/oc +++ b/contrib/completions/zsh/oc @@ -698,6 +698,10 @@ _oc_adm_catalog_mirror() two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--icsp-scope=") two_word_flags+=("--icsp-scope") local_nonpersistent_flags+=("--icsp-scope") @@ -708,6 +712,8 @@ _oc_adm_catalog_mirror() local_nonpersistent_flags+=("--index-filter-by-os=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--manifests-only") local_nonpersistent_flags+=("--manifests-only") flags+=("--max-components=") @@ -5881,8 +5887,14 @@ _oc_adm_release_extract() two_word_flags+=("--git") local_nonpersistent_flags+=("--git") local_nonpersistent_flags+=("--git=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry") @@ -5993,6 +6005,10 @@ _oc_adm_release_info() two_word_flags+=("--dir") local_nonpersistent_flags+=("--dir") local_nonpersistent_flags+=("--dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--image-for=") two_word_flags+=("--image-for") local_nonpersistent_flags+=("--image-for") @@ -6001,6 +6017,8 @@ _oc_adm_release_info() local_nonpersistent_flags+=("--include-images") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry") @@ -6103,8 +6121,14 @@ _oc_adm_release_mirror() two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry") @@ -6117,6 +6141,10 @@ _oc_adm_release_mirror() local_nonpersistent_flags+=("--registry-config") local_nonpersistent_flags+=("--registry-config=") local_nonpersistent_flags+=("-a") + flags+=("--release-image-icsp-to-dir=") + two_word_flags+=("--release-image-icsp-to-dir") + local_nonpersistent_flags+=("--release-image-icsp-to-dir") + local_nonpersistent_flags+=("--release-image-icsp-to-dir=") flags+=("--release-image-signature-to-dir=") two_word_flags+=("--release-image-signature-to-dir") local_nonpersistent_flags+=("--release-image-signature-to-dir") @@ -6239,12 +6267,18 @@ _oc_adm_release_new() two_word_flags+=("--from-release") local_nonpersistent_flags+=("--from-release") local_nonpersistent_flags+=("--from-release=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--include=") two_word_flags+=("--include") local_nonpersistent_flags+=("--include") local_nonpersistent_flags+=("--include=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--mapping-file=") two_word_flags+=("--mapping-file") local_nonpersistent_flags+=("--mapping-file") @@ -14679,12 +14713,18 @@ _oc_image_append() two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--image=") two_word_flags+=("--image") local_nonpersistent_flags+=("--image") local_nonpersistent_flags+=("--image=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry") @@ -14781,8 +14821,14 @@ _oc_image_extract() two_word_flags+=("--filter-by-os") local_nonpersistent_flags+=("--filter-by-os") local_nonpersistent_flags+=("--filter-by-os=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--only-files") local_nonpersistent_flags+=("--only-files") flags+=("--path=") @@ -14867,8 +14913,14 @@ _oc_image_info() two_word_flags+=("--filter-by-os") local_nonpersistent_flags+=("--filter-by-os") local_nonpersistent_flags+=("--filter-by-os=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--output=") two_word_flags+=("--output") two_word_flags+=("-o") @@ -14965,10 +15017,16 @@ _oc_image_mirror() two_word_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir") local_nonpersistent_flags+=("--from-dir=") + flags+=("--icsp-file=") + two_word_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file") + local_nonpersistent_flags+=("--icsp-file=") flags+=("--insecure") local_nonpersistent_flags+=("--insecure") flags+=("--keep-manifest-list") local_nonpersistent_flags+=("--keep-manifest-list") + flags+=("--lookup-cluster-icsp") + local_nonpersistent_flags+=("--lookup-cluster-icsp") flags+=("--max-per-registry=") two_word_flags+=("--max-per-registry") local_nonpersistent_flags+=("--max-per-registry") diff --git a/go.mod b/go.mod index 8f6fca94ac..4291c91255 100644 --- a/go.mod +++ b/go.mod @@ -35,9 +35,9 @@ require ( github.com/onsi/gomega v1.9.0 // indirect github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6 - github.com/openshift/api v0.0.0-20210105115604-44119421ec6b + github.com/openshift/api v0.0.0-20210331193751-3acddb19d360 github.com/openshift/build-machinery-go v0.0.0-20210209125900-0da259a2c359 - github.com/openshift/client-go v0.0.0-20201214125552-e615e336eb49 + github.com/openshift/client-go v0.0.0-20210331195552-cf6c2669e01f github.com/openshift/library-go v0.0.0-20210219155623-0260bfd7946b github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.9.0 @@ -74,7 +74,7 @@ replace ( // temporary pins to latest commit from soltysh/k8s-1.21 branches github.com/openshift/api => github.com/soltysh/api v0.0.0-20210329125043-97dfec49c179 github.com/openshift/client-go => github.com/soltysh/client-go v0.0.0-20210329131047-e53d4b5c79d3 - github.com/openshift/library-go => github.com/soltysh/library-go v0.0.0-20210329143525-072267446dd3 + github.com/openshift/library-go => /home/somalley/code/gowork/src/github.com/openshift/library-go // temporary pins to latest commit from oc-4.8-kubernetes-1.21.0-beta.1 branches k8s.io/apimachinery => github.com/openshift/kubernetes-apimachinery v0.0.0-20210318140035-c39220d4515a diff --git a/go.sum b/go.sum index c357e16332..a2e76c04d1 100644 --- a/go.sum +++ b/go.sum @@ -746,8 +746,6 @@ github.com/soltysh/api v0.0.0-20210329125043-97dfec49c179 h1:A9IqSWz8m9G1RbLJx2j github.com/soltysh/api v0.0.0-20210329125043-97dfec49c179/go.mod h1:dZ4kytOo3svxJHNYd0J55hwe/6IQG5gAUHUE0F3Jkio= github.com/soltysh/client-go v0.0.0-20210329131047-e53d4b5c79d3 h1:D0nYNGWfWlhkguMtWa8XyF8pMcZRH37zboh34bvs+Ms= github.com/soltysh/client-go v0.0.0-20210329131047-e53d4b5c79d3/go.mod h1:khyjHEjWI0OiW5RY1M8N6YCnFAZ986nw7p2OqSufH2A= -github.com/soltysh/library-go v0.0.0-20210329143525-072267446dd3 h1:JwIGY81TlonJjYIKwK2gxbTkqg8iYOiN+udRpD3jihQ= -github.com/soltysh/library-go v0.0.0-20210329143525-072267446dd3/go.mod h1:lwiTZ+1U938RDJPWuIp5aN+eDP2rn9UV91bRvMoWfYA= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= diff --git a/pkg/cli/admin/catalog/mirror.go b/pkg/cli/admin/catalog/mirror.go index 393115ff3e..eb3b3e429e 100644 --- a/pkg/cli/admin/catalog/mirror.go +++ b/pkg/cli/admin/catalog/mirror.go @@ -116,7 +116,7 @@ func NewMirrorCatalog(f kcmdutil.Factory, streams genericclioptions.IOStreams) * Long: mirrorLong, Example: mirrorExample, Run: func(cmd *cobra.Command, args []string) { - kcmdutil.CheckErr(o.Complete(cmd, args)) + kcmdutil.CheckErr(o.Complete(f, cmd, args)) kcmdutil.CheckErr(o.Validate()) kcmdutil.CheckErr(o.Run()) }, @@ -148,7 +148,7 @@ func NewMirrorCatalog(f kcmdutil.Factory, streams genericclioptions.IOStreams) * return cmd } -func (o *MirrorCatalogOptions) Complete(cmd *cobra.Command, args []string) error { +func (o *MirrorCatalogOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { if len(args) < 2 { return fmt.Errorf("must specify source and dest") } @@ -311,7 +311,7 @@ func (o *MirrorCatalogOptions) Complete(cmd *cobra.Command, args []string) error } e.Paths = []string{o.DatabasePath} e.Confirm = true - if err := e.Complete(cmd, []string{o.SourceRef.String()}); err != nil { + if err := e.Complete(f, cmd, []string{o.SourceRef.String()}); err != nil { return "", err } if err := e.Validate(); err != nil { diff --git a/pkg/cli/admin/release/extract.go b/pkg/cli/admin/release/extract.go index 06d3988cae..82bacddb18 100644 --- a/pkg/cli/admin/release/extract.go +++ b/pkg/cli/admin/release/extract.go @@ -173,6 +173,14 @@ func (o *ExtractOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args [ return fmt.Errorf("you may only specify a single image via --from or argument") } o.From = args[0] + if cmd.Flags().Changed("lookup-cluster-icsp") && !o.SecurityOptions.LookupClusterICSP { + o.SecurityOptions.LookupClusterICSP = false + } else if len(o.SecurityOptions.ICSPFile) == 0 { + o.SecurityOptions.LookupClusterICSP = true + } + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } return nil } diff --git a/pkg/cli/admin/release/extract_tools.go b/pkg/cli/admin/release/extract_tools.go index 5e1121a6f5..35ad370d16 100644 --- a/pkg/cli/admin/release/extract_tools.go +++ b/pkg/cli/admin/release/extract_tools.go @@ -27,7 +27,6 @@ import ( "k8s.io/klog/v2" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/cli-runtime/pkg/genericclioptions" "github.com/MakeNowJust/heredoc" imagereference "github.com/openshift/library-go/pkg/image/reference" @@ -320,11 +319,11 @@ func (o *ExtractOptions) extractCommand(command string) error { missing.Insert(target.Mapping.Image) continue } - klog.V(2).Infof("Will extract %s from %s", target.Mapping.From, spec) ref, err := imagereference.Parse(spec) if err != nil { return err } + klog.V(2).Infof("Will extract %s from %s", target.Mapping.From, spec) target.Mapping.Image = spec target.Mapping.ImageRef = imagesource.TypedImageReference{Ref: ref, Type: imagesource.DestinationRegistry} if target.AsArchive { @@ -349,7 +348,7 @@ func (o *ExtractOptions) extractCommand(command string) error { } // will extract in parallel - opts := extract.NewExtractOptions(genericclioptions.IOStreams{Out: o.Out, ErrOut: o.ErrOut}) + opts := extract.NewExtractOptions(o.IOStreams) opts.ParallelOptions = o.ParallelOptions opts.SecurityOptions = o.SecurityOptions opts.OnlyFiles = true diff --git a/pkg/cli/admin/release/image_mapper.go b/pkg/cli/admin/release/image_mapper.go index 7adc91b661..c193a2daa8 100644 --- a/pkg/cli/admin/release/image_mapper.go +++ b/pkg/cli/admin/release/image_mapper.go @@ -2,6 +2,7 @@ package release import ( "bytes" + "context" "fmt" "io/ioutil" "os" @@ -12,8 +13,10 @@ import ( "github.com/blang/semver" "github.com/ghodss/yaml" + "github.com/opencontainers/go-digest" imageapi "github.com/openshift/api/image/v1" imagereference "github.com/openshift/library-go/pkg/image/reference" + registryclient "github.com/openshift/library-go/pkg/image/registryclient" "k8s.io/klog/v2" ) @@ -103,10 +106,11 @@ func parseImageStream(path string) (*imageapi.ImageStream, error) { if err != nil { return nil, fmt.Errorf("unable to read release image info from release contents: %v", err) } - return readReleaseImageReferences(data) + return readReleaseImageReferences(nil, data, imagereference.DockerImageReference{}, false) } -func readReleaseImageReferences(data []byte) (*imageapi.ImageStream, error) { +func readReleaseImageReferences(regContext *registryclient.Context, data []byte, newRef imagereference.DockerImageReference, insecure bool) (*imageapi.ImageStream, error) { + ctx := context.Background() is := &imageapi.ImageStream{} if err := yaml.Unmarshal(data, &is); err != nil { return nil, fmt.Errorf("unable to load release image-references: %v", err) @@ -114,6 +118,52 @@ func readReleaseImageReferences(data []byte) (*imageapi.ImageStream, error) { if is.Kind != "ImageStream" || is.APIVersion != "image.openshift.io/v1" { return nil, fmt.Errorf("unrecognized image-references in release payload") } + + if regContext == nil { + return is, nil + } + tagRef, err := imagereference.Parse(is.Spec.Tags[0].From.Name) + if err != nil { + return nil, err + } + // Only want to use the replaced reference if the tagRef is not accessible. + // With mirrored release, you would want to use the user-passed, ex: use localhost:5000 not the tagRef: quay.io/openshift-release-dev/ocp-*-art-dev + // However, with a nightly release (ex: registry.ci.openshift.org) you want to use the tagRef: quay.io/openshift-release-dev/ocp-*-art-dev + // and not the user-passed registry.ci.openshift.org. + // If tagRef is accessible, return without replacing with the newRef. + repo, err := regContext.RepositoryForRef(ctx, tagRef, insecure) + if err == nil { + manifests, err := repo.Manifests(ctx) + if err == nil { + dgst := digest.Digest(tagRef.ID) + _, err = manifests.Get(ctx, dgst) + } + if err == nil { + return is, nil + } + } + klog.V(4).Infof("Could not access image tag reference: %s, %v. Will try with replaced reference: %s", tagRef.AsRepository().String(), err, newRef.AsRepository().String()) + + // ensure image reference tags are updated with preferred image reference, update when: + // 1) mirrored release, extract from the mirrored release name ex: private:5000/release as opposed to quay.io/openshift-release-dev + for i, tag := range is.Spec.Tags { + if tag.From == nil || tag.From.Kind != "DockerImage" { + continue + } + tRef, err := imagereference.Parse(tag.From.Name) + if err != nil { + continue + } + // ie don't update if this image reference is a substituted component image + if tag.Annotations != nil { + if _, ok := tag.Annotations[annotationReleaseOverride]; ok { + continue + } + newRef.ID = tRef.ID + newRef.Tag = tRef.Tag + is.Spec.Tags[i].From.Name = newRef.String() + } + } return is, nil } diff --git a/pkg/cli/admin/release/info.go b/pkg/cli/admin/release/info.go index f31e0b0c74..8d2cc150e9 100644 --- a/pkg/cli/admin/release/info.go +++ b/pkg/cli/admin/release/info.go @@ -367,6 +367,16 @@ func (o *InfoOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []st o.From = o.Images[0] o.Images = o.Images[1:] } + + if cmd.Flags().Changed("lookup-cluster-icsp") && !o.SecurityOptions.LookupClusterICSP { + o.SecurityOptions.LookupClusterICSP = false + } else if len(o.SecurityOptions.ICSPFile) == 0 { + o.SecurityOptions.LookupClusterICSP = true + } + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } + return nil } @@ -734,17 +744,33 @@ func (i *ReleaseInfo) Platform() string { return fmt.Sprintf("%s/%s", os, arch) } +// LoadReleaseInfo takes an image and returns release payload info or an error. +// If retrieveImages is true, release payload component images will also be fetched. func (o *InfoOptions) LoadReleaseInfo(image string, retrieveImages bool) (*ReleaseInfo, error) { + ctx := context.Background() ref, err := imagesource.ParseReference(image) if err != nil { return nil, err } - + regContext, err := o.SecurityOptions.Context() + if err != nil { + return nil, err + } + sourceOpts := &imagesource.Options{ + FileDir: o.FileDir, + Insecure: o.SecurityOptions.Insecure, + RegistryContext: regContext, + } + newRef, _, err := sourceOpts.RepositoryWithLocation(ctx, ref) + if err != nil { + return nil, err + } + image = newRef.String() + ref.Ref = newRef verifier := imagemanifest.NewVerifier() - opts := extract.NewExtractOptions(genericclioptions.IOStreams{Out: o.Out, ErrOut: o.ErrOut}) + opts := extract.NewExtractOptions(o.IOStreams) opts.SecurityOptions = o.SecurityOptions opts.FileDir = o.FileDir - release := &ReleaseInfo{ Image: image, ImageRef: ref, @@ -778,7 +804,7 @@ func (o *InfoOptions) LoadReleaseInfo(image string, retrieveImages bool) (*Relea return true, nil } release.RawMetadata[hdr.Name] = data - is, err := readReleaseImageReferences(data) + is, err := readReleaseImageReferences(regContext, data, newRef, opts.SecurityOptions.Insecure) if err != nil { errs = append(errs, err) return true, nil @@ -825,7 +851,6 @@ func (o *InfoOptions) LoadReleaseInfo(image string, retrieveImages bool) (*Relea if release.References == nil { return nil, fmt.Errorf("release image did not contain an image-references file") } - release.ComponentVersions, errs = readComponentVersions(release.References) for _, err := range errs { release.Warnings = append(release.Warnings, err.Error()) diff --git a/pkg/cli/admin/release/mirror.go b/pkg/cli/admin/release/mirror.go index 31b377ebe0..caf5fed79b 100644 --- a/pkg/cli/admin/release/mirror.go +++ b/pkg/cli/admin/release/mirror.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "io/ioutil" "net/http" "os" @@ -50,8 +49,9 @@ import ( ) // configFilesBaseDir is created under '--to-dir', when specified, to contain release image -// signature files. It is not used when '--release-image-signature-to-dir` is specified -// which takes precedence over '--to-dir'. +// signature and icsp files. It is not used for signature file when '--release-image-signature-to-dir' is specified. +// It is not used for icsp file when '--release-image-icsp-to-dir' is specified. +// The signature-to and icsp-to flags take precedence over '--to-dir'. const configFilesBaseDir = "config" // maxDigestHashLen is used to truncate digest hash portion before using as part of @@ -66,6 +66,9 @@ var archMap = map[string]string{ "amd64": "x86_64", } +// icspFileNameFmt defines format of the release image ImageContentSourcePolicy file name +const icspFileNameFmt = "icsp-%s-%s.yaml" + // NewMirrorOptions creates the options for mirroring a release. func NewMirrorOptions(streams genericclioptions.IOStreams) *MirrorOptions { return &MirrorOptions{ @@ -119,22 +122,28 @@ func NewMirror(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.C The --overwrite option only applies when --apply-release-image-signature is specified and indicates to update an exisiting config map if one is found. A config map written to a directory will always replace onethat already exists. + + You may use --release-image-icsp-to-dir to specifiy the export target directory of the + ImageContentSourcePolicy file. The default is 'config'. If not specified but --to-dir is, + --release-image-icsp-to-dir defaults to a 'config' subdirectory of --to-dir. `), Example: templates.Examples(` - # Perform a dry run showing what would be mirrored, including the mirror objects + # Perform a dry run showing what would be mirrored, including the mirror objects, and control where signature and ICSP files are written oc adm release mirror 4.3.0 --to myregistry.local/openshift/release \ - --release-image-signature-to-dir /tmp/releases --dry-run + --release-image-signature-to-dir /tmp/releases \ + --release-image-icsp-to-dir /tmp/icsps --dry-run - # Mirror a release into the current directory + # Mirror a release into the current directory and control where signature and ICSP files are written oc adm release mirror 4.3.0 --to file://openshift/release \ - --release-image-signature-to-dir /tmp/releases + --release-image-signature-to-dir /tmp/releases \ + --release-image-icsp-to-dir /tmp/icsps # Mirror a release to another directory in the default location oc adm release mirror 4.3.0 --to-dir /tmp/releases # Upload a release from the current directory to another server oc adm release mirror --from file://openshift/release --to myregistry.com/openshift/release \ - --release-image-signature-to-dir /tmp/releases + --release-image-config-dir /tmp/releases # Mirror the 4.3.0 release to repository registry.example.com and apply signatures to connected cluster oc adm release mirror --from=quay.io/openshift-release-dev/ocp-release:4.3.0-x86_64 \ @@ -155,10 +164,11 @@ func NewMirror(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.C flags.StringVar(&o.ToImageStream, "to-image-stream", o.ToImageStream, "An image stream to tag images into.") flags.StringVar(&o.FromDir, "from-dir", o.FromDir, "A directory to import images from.") flags.StringVar(&o.ToDir, "to-dir", o.ToDir, "A directory to export images to.") + flags.StringVar(&o.ReleaseImageICSPToDir, "release-image-icsp-to-dir", o.ReleaseImageICSPToDir, "Path to write ImageContentSourcePolicy file. If not set, defaults to 'config'.") + flags.StringVar(&o.ReleaseImageSignatureToDir, "release-image-signature-to-dir", o.ReleaseImageSignatureToDir, "Path to write release image signature ConfigMap files. If not set, defaults to 'config'.") flags.BoolVar(&o.ToMirror, "to-mirror", o.ToMirror, "Output the mirror mappings instead of mirroring.") flags.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Display information about the mirror without actually executing it.") flags.BoolVar(&o.ApplyReleaseImageSignature, "apply-release-image-signature", o.ApplyReleaseImageSignature, "Apply release image signature to connected cluster.") - flags.StringVar(&o.ReleaseImageSignatureToDir, "release-image-signature-to-dir", o.ReleaseImageSignatureToDir, "A directory to export release image signature to.") flags.BoolVar(&o.SkipRelease, "skip-release-image", o.SkipRelease, "Do not push the release image.") flags.StringVar(&o.ToRelease, "to-release-image", o.ToRelease, "Specify an alternate locations for the release image instead as tag 'release' in --to.") @@ -187,10 +197,10 @@ type MirrorOptions struct { ApplyReleaseImageSignature bool ReleaseImageSignatureToDir string + ReleaseImageICSPToDir string Overwrite bool - DryRun bool - PrintImageContentInstructions bool + DryRun bool ImageClientFn func() (imageclient.Interface, string, error) CoreV1ClientFn func() (corev1client.ConfigMapInterface, error) @@ -220,6 +230,15 @@ func (o *MirrorOptions) Complete(cmd *cobra.Command, f kcmdutil.Factory, args [] } o.From = args[0] + if cmd.Flags().Changed("lookup-cluster-icsp") && !o.SecurityOptions.LookupClusterICSP { + o.SecurityOptions.LookupClusterICSP = false + } else if len(o.SecurityOptions.ICSPFile) == 0 { + o.SecurityOptions.LookupClusterICSP = true + } + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } + o.ImageClientFn = func() (imageclient.Interface, string, error) { cfg, err := f.ToRESTConfig() if err != nil { @@ -247,7 +266,7 @@ func (o *MirrorOptions) Complete(cmd *cobra.Command, f kcmdutil.Factory, args [] client := coreClient.ConfigMaps(configmap.NamespaceLabelConfigMap) return client, nil } - o.PrintImageContentInstructions = true + return nil } @@ -285,6 +304,10 @@ func (o *MirrorOptions) Validate() error { o.ReleaseImageSignatureToDir = filepath.Join(o.ToDir, configFilesBaseDir) } + if len(o.ReleaseImageICSPToDir) == 0 && len(o.ToDir) > 0 { + o.ReleaseImageICSPToDir = filepath.Join(o.ToDir, configFilesBaseDir) + } + if o.Overwrite && !o.ApplyReleaseImageSignature { return fmt.Errorf("--overwite is only valid when --apply-release-image-signature is specified") } @@ -317,6 +340,19 @@ func createSignatureFileName(digest string) (string, error) { return fmt.Sprintf(signatureFileNameFmt, algo, hash), nil } +func createICSPFileName(digest string) (string, error) { + parts := strings.SplitN(digest, ":", 3) + if len(parts) != 2 || len(parts[0]) == 0 || len(parts[1]) == 0 { + return "", fmt.Errorf("the provided digest, %s, must be of the form ALGO:HASH", digest) + } + algo, hash := parts[0], parts[1] + + if len(hash) > maxDigestHashLen { + hash = hash[:maxDigestHashLen] + } + return fmt.Sprintf(icspFileNameFmt, algo, hash), nil +} + // handleSignatures implements the image release signature configmap specific logic. // Signature configmaps may be written to a directory or applied to a cluster. func (o *MirrorOptions) handleSignatures(context context.Context, signaturesByDigest map[string][][]byte) error { @@ -803,6 +839,13 @@ func (o *MirrorOptions) Run() error { fmt.Fprintf(o.Out, "Mirrored to: %s\n", t) } } + if len(o.ReleaseImageICSPToDir) == 0 { + o.ReleaseImageICSPToDir = configFilesBaseDir + } + + if err := o.printImageContentInstructions(repositories, toList, releaseDigest); err != nil { + return fmt.Errorf("Error creating mirror usage instructions: %v", err) + } } if toDisk { if len(o.ToDir) > 0 { @@ -810,12 +853,6 @@ func (o *MirrorOptions) Run() error { } else { fmt.Fprintf(o.Out, "\nTo upload local images to a registry, run:\n\n oc image mirror 'file://%s*' REGISTRY/REPOSITORY\n\n", to) } - } else if len(toList) > 0 { - if o.PrintImageContentInstructions { - if err := printImageContentInstructions(o.Out, o.From, toList, o.ReleaseImageSignatureToDir, repositories); err != nil { - return fmt.Errorf("Error creating mirror usage instructions: %v", err) - } - } } if o.ApplyReleaseImageSignature || len(o.ReleaseImageSignatureToDir) > 0 { signatures := imageVerifier.Signatures() @@ -841,15 +878,17 @@ func (o *MirrorOptions) Run() error { // printImageContentInstructions provides examples to the user for using the new repository mirror // https://github.com/openshift/installer/blob/master/docs/dev/alternative_release_image_sources.md -func printImageContentInstructions(out io.Writer, from string, toList []string, signatureToDir string, repositories map[string]struct{}) error { +func (o *MirrorOptions) printImageContentInstructions(repositories map[string]struct{}, toList []string, digest string) error { type installConfigSubsection struct { ImageContentSources []operatorv1alpha1.RepositoryDigestMirrors `json:"imageContentSources"` } var sources []operatorv1alpha1.RepositoryDigestMirrors + var mirrorRef imagesource.TypedImageReference + var err error for _, to := range toList { - mirrorRef, err := imagesource.ParseReference(to) + mirrorRef, err = imagesource.ParseReference(to) if err != nil { return fmt.Errorf("Unable to parse image reference '%s': %v", to, err) } @@ -857,10 +896,11 @@ func printImageContentInstructions(out io.Writer, from string, toList []string, return nil } mirrorRepo := mirrorRef.Ref.AsRepository().String() - if len(from) != 0 { - sourceRef, err := imagesource.ParseReference(from) + + if len(o.From) != 0 { + sourceRef, err := imagesource.ParseReference(o.From) if err != nil { - return fmt.Errorf("Unable to parse image reference '%s': %v", from, err) + return fmt.Errorf("Unable to parse image reference '%s': %v", o.From, err) } if sourceRef.Type != imagesource.DestinationRegistry { return nil @@ -868,7 +908,6 @@ func printImageContentInstructions(out io.Writer, from string, toList []string, sourceRepo := sourceRef.Ref.AsRepository().String() repositories[sourceRepo] = struct{}{} } - if len(repositories) == 0 { return nil } @@ -891,16 +930,20 @@ func printImageContentInstructions(out io.Writer, from string, toList []string, if err != nil { return fmt.Errorf("Unable to marshal install-config.yaml example yaml: %v", err) } - fmt.Fprintf(out, "\nTo use the new mirrored repository to install, add the following section to the install-config.yaml:\n\n") - fmt.Fprintf(out, string(installConfigExample)) + fmt.Fprintf(o.Out, "\nTo use the new mirrored repository to install, add the following section to the install-config.yaml:\n\n") + fmt.Fprintf(o.Out, string(installConfigExample)) + + // Create and display ImageContentSourcePolicy + // last mirrorRef will be used to name icsp + mirrorRepoStripped := strings.FieldsFunc(mirrorRef.Ref.Name, func(r rune) bool { return strings.ContainsRune(" .:/", r) }) + icspName := strings.Join(mirrorRepoStripped[:], "-") - // Create and display ImageContentSourcePolicy example icsp := operatorv1alpha1.ImageContentSourcePolicy{ TypeMeta: metav1.TypeMeta{ APIVersion: operatorv1alpha1.GroupVersion.String(), Kind: "ImageContentSourcePolicy"}, ObjectMeta: metav1.ObjectMeta{ - Name: "example", + Name: icspName, }, Spec: operatorv1alpha1.ImageContentSourcePolicySpec{ RepositoryDigestMirrors: sources, @@ -915,16 +958,30 @@ func printImageContentInstructions(out io.Writer, from string, toList []string, } delete(unstructuredObj.Object["metadata"].(map[string]interface{}), "creationTimestamp") - icspExample, err := yaml.Marshal(unstructuredObj.Object) + icspDataBytes, err := yaml.Marshal(unstructuredObj.Object) if err != nil { - return fmt.Errorf("Unable to marshal ImageContentSourcePolicy example yaml: %v", err) + return fmt.Errorf("Unable to marshal ImageContentSourcePolicy yaml: %v", err) } - fmt.Fprintf(out, "\n\nTo use the new mirrored repository for upgrades, use the following to create an ImageContentSourcePolicy:\n\n") - fmt.Fprintf(out, string(icspExample)) - - if len(signatureToDir) != 0 { - fmt.Fprintf(out, "\n\nTo apply signature configmaps use 'oc apply' on files found in %s\n\n", signatureToDir) + icspFileName, err := createICSPFileName(digest) + if err != nil { + return fmt.Errorf("creating filename: %v", err) } + icspFullName := filepath.Join(o.ReleaseImageICSPToDir, icspFileName) + if o.DryRun { + fmt.Fprintf(o.Out, "\ninfo: Write ImageContentSourcePolicy file %s\n", icspFullName) + } else { + if err := os.MkdirAll(filepath.Dir(icspFullName), 0750); err != nil { + return err + } + if err := ioutil.WriteFile(icspFullName, icspDataBytes, 0640); err != nil { + return err + } + fmt.Fprintf(o.Out, "\nImageContentSourcePolicy file %s created\n", icspFullName) + } + fmt.Fprintf(o.Out, "\n\nTo use the new mirrored repository for upgrades, use the following to create an ImageContentSourcePolicy:\n\n") + fmt.Fprintf(o.Out, string(icspDataBytes)) + + fmt.Fprintf(o.Out, "\n\nTo apply ImageContentSourcePolicy, use 'oc apply' on file %s\n\n", icspFullName) return nil } diff --git a/pkg/cli/admin/release/new.go b/pkg/cli/admin/release/new.go index 65fdb3ec03..4e04e37ae7 100644 --- a/pkg/cli/admin/release/new.go +++ b/pkg/cli/admin/release/new.go @@ -246,6 +246,15 @@ func (o *NewOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []str o.Namespace = namespace } } + if cmd.Flags().Changed("lookup-cluster-icsp") && !o.SecurityOptions.LookupClusterICSP { + o.SecurityOptions.LookupClusterICSP = false + } else if len(o.SecurityOptions.ICSPFile) == 0 { + o.SecurityOptions.LookupClusterICSP = true + } + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } + return nil } @@ -441,7 +450,7 @@ func (o *NewOptions) Run() error { fmt.Fprintf(o.ErrOut, "warning: %v\n", err) } - inputIS, err := readReleaseImageReferences(imageReferencesData) + inputIS, err := readReleaseImageReferences(nil, imageReferencesData, ref.Ref, false) if err != nil { return fmt.Errorf("unable to load image-references from release contents: %v", err) } diff --git a/pkg/cli/image/append/append.go b/pkg/cli/image/append/append.go index 752f1bd585..ff18d4c0a2 100644 --- a/pkg/cli/image/append/append.go +++ b/pkg/cli/image/append/append.go @@ -126,7 +126,7 @@ func NewAppendImageOptions(streams genericclioptions.IOStreams) *AppendImageOpti } // New creates a new command -func NewCmdAppendImage(streams genericclioptions.IOStreams) *cobra.Command { +func NewCmdAppendImage(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewAppendImageOptions(streams) cmd := &cobra.Command{ @@ -135,7 +135,7 @@ func NewCmdAppendImage(streams genericclioptions.IOStreams) *cobra.Command { Long: desc, Example: example, Run: func(c *cobra.Command, args []string) { - kcmdutil.CheckErr(o.Complete(c, args)) + kcmdutil.CheckErr(o.Complete(f, c, args)) kcmdutil.CheckErr(o.Validate()) kcmdutil.CheckErr(o.Run()) }, @@ -164,11 +164,13 @@ func NewCmdAppendImage(streams genericclioptions.IOStreams) *cobra.Command { return cmd } -func (o *AppendImageOptions) Complete(cmd *cobra.Command, args []string) error { +func (o *AppendImageOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { if err := o.FilterOptions.Complete(cmd.Flags()); err != nil { return err } - + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } for _, arg := range args { if arg == "-" { if o.LayerStream != nil { @@ -267,16 +269,19 @@ func (o *AppendImageOptions) Run() error { fromRepo distribution.Repository ) if from != nil { - repo, err := fromOptions.Repository(ctx, *from) + newRef, repo, err := fromOptions.RepositoryWithLocation(ctx, *from) if err != nil { return err } fromRepo = repo + from.Ref = newRef - srcManifest, manifestLocation, err := imagemanifest.FirstManifest(ctx, from.Ref, repo, o.FilterOptions.Include) + newRef, srcManifest, manifestLocation, err := imagemanifest.FirstManifest(ctx, from.Ref, fromRepo.(registryclient.RepositoryWithLocation), o.FilterOptions.Include) if err != nil { return fmt.Errorf("unable to read image %s: %v", from, err) } + + from.Ref = newRef base, layers, err = imagemanifest.ManifestToImageConfig(ctx, srcManifest, repo.Blobs(ctx), manifestLocation) if err != nil { return fmt.Errorf("unable to parse image %s: %v", from, err) diff --git a/pkg/cli/image/extract/extract.go b/pkg/cli/image/extract/extract.go index 4397ab2d5a..dfc3e5c0f7 100644 --- a/pkg/cli/image/extract/extract.go +++ b/pkg/cli/image/extract/extract.go @@ -156,7 +156,7 @@ func NewExtractOptions(streams genericclioptions.IOStreams) *ExtractOptions { } // New creates a new command -func NewExtract(streams genericclioptions.IOStreams) *cobra.Command { +func NewExtract(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewExtractOptions(streams) cmd := &cobra.Command{ @@ -165,7 +165,7 @@ func NewExtract(streams genericclioptions.IOStreams) *cobra.Command { Long: desc, Example: example, Run: func(c *cobra.Command, args []string) { - kcmdutil.CheckErr(o.Complete(c, args)) + kcmdutil.CheckErr(o.Complete(f, c, args)) kcmdutil.CheckErr(o.Validate()) kcmdutil.CheckErr(o.Run()) }, @@ -307,7 +307,7 @@ func parseMappings(images, paths, files []string, requireEmpty bool) ([]Mapping, return mappings, nil } -func (o *ExtractOptions) Complete(cmd *cobra.Command, args []string) error { +func (o *ExtractOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { if err := o.FilterOptions.Complete(cmd.Flags()); err != nil { return err } @@ -325,6 +325,11 @@ func (o *ExtractOptions) Complete(cmd *cobra.Command, args []string) error { if err != nil { return err } + + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } + return nil } @@ -355,15 +360,14 @@ func (o *ExtractOptions) Run() error { mapping := o.Mappings[i] from := mapping.ImageRef q.Try(func() error { - repo, err := fromOptions.Repository(ctx, from) + newRef, repo, err := fromOptions.RepositoryWithLocation(ctx, from) if err != nil { return fmt.Errorf("unable to connect to image repository %s: %v", from.String(), err) } - - srcManifest, location, err := imagemanifest.FirstManifest(ctx, from.Ref, repo, o.FilterOptions.Include) + newRef, srcManifest, location, err := imagemanifest.FirstManifest(ctx, from.Ref, repo, o.FilterOptions.Include) if err != nil { if imagemanifest.IsImageForbidden(err) { - msg := fmt.Sprintf("image %q does not exist or you don't have permission to access the repository", from) + msg := fmt.Sprintf("image %q does not exist or you don't have permission to access the repository", from.Ref.String()) return imagemanifest.NewImageForbidden(msg, err) } if imagemanifest.IsImageNotFound(err) { @@ -373,10 +377,12 @@ func (o *ExtractOptions) Run() error { return fmt.Errorf("unable to read image %s: %v", from, err) } + from.Ref = newRef contentDigest, err := registryclient.ContentDigestForManifest(srcManifest, location.Manifest.Algorithm()) if err != nil { return err } + _, repo, err = fromOptions.RepositoryWithLocation(ctx, from) imageConfig, layers, err := imagemanifest.ManifestToImageConfig(ctx, srcManifest, repo.Blobs(ctx), location) if err != nil { @@ -486,7 +492,7 @@ func (o *ExtractOptions) Run() error { if byEntry != nil { cont, err := layerByEntry(r, options, info, byEntry, o.AllLayers, alreadySeen) if err != nil { - err = fmt.Errorf("unable to iterate over layer %s from %s: %v", layer.Digest, from, err) + err = fmt.Errorf("HERERERER unable to iterate over layer %s from %s: %v", layer.Digest, from, err) } return cont, err } diff --git a/pkg/cli/image/image.go b/pkg/cli/image/image.go index 48d9a8351b..f6d860c176 100644 --- a/pkg/cli/image/image.go +++ b/pkg/cli/image/image.go @@ -35,16 +35,16 @@ func NewCmdImage(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra { Message: "View or copy images:", Commands: []*cobra.Command{ - info.NewInfo(streams), - mirror.NewCmdMirrorImage(streams), + info.NewInfo(f, streams), + mirror.NewCmdMirrorImage(f, streams), }, }, { Message: "Advanced commands:", Commands: []*cobra.Command{ serve.NewServe(streams), - append.NewCmdAppendImage(streams), - extract.NewExtract(streams), + append.NewCmdAppendImage(f, streams), + extract.NewExtract(f, streams), }, }, } diff --git a/pkg/cli/image/imagesource/options.go b/pkg/cli/image/imagesource/options.go index 4662b0e680..1ee002ad88 100644 --- a/pkg/cli/image/imagesource/options.go +++ b/pkg/cli/image/imagesource/options.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/docker/distribution" + imagereference "github.com/openshift/library-go/pkg/image/reference" "github.com/openshift/library-go/pkg/image/registryclient" "k8s.io/klog/v2" ) @@ -17,6 +18,21 @@ type Options struct { RegistryContext *registryclient.Context } +// RepositoryWithLocation retrieves the appropriate repository implementation and imageReference for the given typed reference. +func (o *Options) RepositoryWithLocation(ctx context.Context, ref TypedImageReference) (imagereference.DockerImageReference, registryclient.RepositoryWithLocation, error) { + if ref.Type == DestinationRegistry { + repo, err := o.RegistryContext.Repository(ctx, ref.Ref.DockerClientDefaults().RegistryURL(), ref.Ref.RepositoryName(), o.Insecure) + if err != nil { + return ref.Ref, nil, err + } + newRef := repo.Ref() + newRef.ID = ref.Ref.ID + newRef.Tag = ref.Ref.Tag + return newRef, repo, nil + } + return ref.Ref, nil, fmt.Errorf("expected image reference of type registry %s, use o.Repository instead.", ref.Type) +} + // Repository retrieves the appropriate repository implementation for the given typed reference. func (o *Options) Repository(ctx context.Context, ref TypedImageReference) (distribution.Repository, error) { switch ref.Type { diff --git a/pkg/cli/image/info/info.go b/pkg/cli/image/info/info.go index 7aa9f3d201..17c7d9a9f7 100644 --- a/pkg/cli/image/info/info.go +++ b/pkg/cli/image/info/info.go @@ -38,7 +38,7 @@ func NewInfoOptions(streams genericclioptions.IOStreams) *InfoOptions { } } -func NewInfo(streams genericclioptions.IOStreams) *cobra.Command { +func NewInfo(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewInfoOptions(streams) cmd := &cobra.Command{ Use: "info IMAGE [...]", @@ -68,7 +68,7 @@ func NewInfo(streams genericclioptions.IOStreams) *cobra.Command { `), Run: func(cmd *cobra.Command, args []string) { - kcmdutil.CheckErr(o.Complete(cmd, args)) + kcmdutil.CheckErr(o.Complete(f, cmd, args)) kcmdutil.CheckErr(o.Validate()) kcmdutil.CheckErr(o.Run()) }, @@ -94,11 +94,14 @@ type InfoOptions struct { Output string } -func (o *InfoOptions) Complete(cmd *cobra.Command, args []string) error { +func (o *InfoOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { if len(args) < 1 { return fmt.Errorf("info expects at least one argument, an image pull spec") } o.Images = args + if err := o.SecurityOptions.Complete(f); err != nil { + return err + } return nil } @@ -403,12 +406,12 @@ func (o *ImageRetriever) Images(ctx context.Context, refs map[string]imagesource name := key from := refs[key] q.Try(func() error { - repo, err := fromOptions.Repository(ctx, from) + newRef, repo, err := fromOptions.RepositoryWithLocation(ctx, from) if err != nil { return callbackFn(name, nil, fmt.Errorf("unable to connect to image repository %s: %v", from, err)) } - - allManifests, manifestList, listDigest, err := imagemanifest.AllManifests(ctx, from.Ref, repo) + from.Ref = newRef + newRef, allManifests, manifestList, listDigest, err := imagemanifest.AllManifests(ctx, from.Ref, repo) if err != nil { if imagemanifest.IsImageForbidden(err) { msg := fmt.Sprintf("image %q does not exist or you don't have permission to access the repository", from) @@ -421,6 +424,7 @@ func (o *ImageRetriever) Images(ctx context.Context, refs map[string]imagesource return callbackFn(name, nil, fmt.Errorf("unable to read image %s: %v", from, err)) } + from.Ref = newRef if o.ManifestListCallback != nil && manifestList != nil { allManifests, err = o.ManifestListCallback(name, manifestList, allManifests) if err != nil { @@ -441,7 +445,7 @@ func (o *ImageRetriever) Images(ctx context.Context, refs map[string]imagesource imageConfig, layers, manifestErr := imagemanifest.ManifestToImageConfig(ctx, srcManifest, repo.Blobs(ctx), imagemanifest.ManifestLocation{ManifestList: listDigest, Manifest: srcDigest}) mediaType, _, _ := srcManifest.Payload() if err := callbackFn(name, &Image{ - Name: from.Ref.Exact(), + Name: newRef.Exact(), Ref: from, MediaType: mediaType, Digest: srcDigest, diff --git a/pkg/cli/image/manifest/manifest.go b/pkg/cli/image/manifest/manifest.go index 283ed1e1cd..812b74bb19 100644 --- a/pkg/cli/image/manifest/manifest.go +++ b/pkg/cli/image/manifest/manifest.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "net/http" "regexp" "runtime" "sync" @@ -22,14 +21,12 @@ import ( "github.com/docker/libtrust" "github.com/opencontainers/go-digest" - "k8s.io/client-go/rest" "k8s.io/klog/v2" imagespecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/openshift/library-go/pkg/image/dockerv1client" imagereference "github.com/openshift/library-go/pkg/image/reference" "github.com/openshift/library-go/pkg/image/registryclient" - "github.com/openshift/oc/pkg/cli/image/manifest/dockercredentials" "github.com/openshift/oc/pkg/helpers/image/dockerlayer/add" ) @@ -41,37 +38,6 @@ func (o *ParallelOptions) Bind(flags *pflag.FlagSet) { flags.IntVar(&o.MaxPerRegistry, "max-per-registry", o.MaxPerRegistry, "Number of concurrent requests allowed per registry.") } -type SecurityOptions struct { - RegistryConfig string - Insecure bool - SkipVerification bool - - CachedContext *registryclient.Context -} - -func (o *SecurityOptions) Bind(flags *pflag.FlagSet) { - flags.StringVarP(&o.RegistryConfig, "registry-config", "a", o.RegistryConfig, "Path to your registry credentials (defaults to ~/.docker/config.json)") - flags.BoolVar(&o.Insecure, "insecure", o.Insecure, "Allow push and pull operations to registries to be made over HTTP") - flags.BoolVar(&o.SkipVerification, "skip-verification", o.SkipVerification, "Skip verifying the integrity of the retrieved content. This is not recommended, but may be necessary when importing images from older image registries. Only bypass verification if the registry is known to be trustworthy.") -} - -// ReferentialHTTPClient returns an http.Client that is appropriate for accessing -// blobs referenced outside of the registry (due to the present of the URLs attribute -// in the manifest reference for a layer). -func (o *SecurityOptions) ReferentialHTTPClient() (*http.Client, error) { - ctx, err := o.Context() - if err != nil { - return nil, err - } - client := &http.Client{} - if o.Insecure { - client.Transport = ctx.InsecureTransport - } else { - client.Transport = ctx.Transport - } - return client, nil -} - type Verifier interface { Verify(dgst, contentDgst digest.Digest) Verified() bool @@ -101,39 +67,6 @@ func (v *verifier) Verified() bool { return !v.hadError } -func (o *SecurityOptions) Context() (*registryclient.Context, error) { - if o.CachedContext != nil { - return o.CachedContext, nil - } - context, err := o.NewContext() - if err == nil { - o.CachedContext = context - o.CachedContext.Retries = 3 - } - return context, err -} - -func (o *SecurityOptions) NewContext() (*registryclient.Context, error) { - rt, err := rest.TransportFor(&rest.Config{}) - if err != nil { - return nil, err - } - insecureRT, err := rest.TransportFor(&rest.Config{TLSClientConfig: rest.TLSClientConfig{Insecure: true}}) - if err != nil { - return nil, err - } - creds := dockercredentials.NewLocal() - if len(o.RegistryConfig) > 0 { - creds, err = dockercredentials.NewFromFile(o.RegistryConfig) - if err != nil { - return nil, fmt.Errorf("unable to load --registry-config: %v", err) - } - } - context := registryclient.NewContext(rt, insecureRT).WithCredentials(creds) - context.DisableDigestVerification = o.SkipVerification - return context, nil -} - // FilterOptions assist in filtering out unneeded manifests from ManifestList objects. type FilterOptions struct { FilterByOS string @@ -217,29 +150,33 @@ var PreferManifestList = distribution.WithManifestMediaTypes([]string{ }) // AllManifests returns all non-list manifests, the list manifest (if any), the digest the from refers to, or an error. -func AllManifests(ctx context.Context, from imagereference.DockerImageReference, repo distribution.Repository) (map[digest.Digest]distribution.Manifest, *manifestlist.DeserializedManifestList, digest.Digest, error) { +func AllManifests(ctx context.Context, from imagereference.DockerImageReference, repo distribution.Repository) (imagereference.DockerImageReference, map[digest.Digest]distribution.Manifest, *manifestlist.DeserializedManifestList, digest.Digest, error) { var srcDigest digest.Digest if len(from.ID) > 0 { srcDigest = digest.Digest(from.ID) } else if len(from.Tag) > 0 { desc, err := repo.Tags(ctx).Get(ctx, from.Tag) if err != nil { - return nil, nil, "", err + return from, nil, nil, "", err } srcDigest = desc.Digest } else { - return nil, nil, "", fmt.Errorf("no tag or digest specified") + return from, nil, nil, "", fmt.Errorf("no tag or digest specified") } manifests, err := repo.Manifests(ctx) if err != nil { - return nil, nil, "", err + return from, nil, nil, "", err } - srcManifest, err := manifests.Get(ctx, srcDigest, PreferManifestList) + srcManifest, ref, err := manifests.(registryclient.ManifestWithLocationService).GetWithLocation(ctx, srcDigest, PreferManifestList) if err != nil { - return nil, nil, "", err + return from, nil, nil, "", err + } + if srcManifest == nil { + return from, nil, nil, "", fmt.Errorf("srcManifest for ref %s is nil", from.String()) } - return ManifestsFromList(ctx, srcDigest, srcManifest, manifests, from) + dMap, ml, dgst, err := ManifestsFromList(ctx, srcDigest, srcManifest, manifests, ref) + return ref, dMap, ml, dgst, err } type ManifestLocation struct { @@ -259,41 +196,44 @@ func (m ManifestLocation) String() string { } // FirstManifest returns the first manifest at the request location that matches the filter function. -func FirstManifest(ctx context.Context, from imagereference.DockerImageReference, repo distribution.Repository, filterFn FilterFunc) (distribution.Manifest, ManifestLocation, error) { +func FirstManifest(ctx context.Context, from imagereference.DockerImageReference, repo registryclient.RepositoryWithLocation, filterFn FilterFunc) (imagereference.DockerImageReference, distribution.Manifest, ManifestLocation, error) { var srcDigest digest.Digest if len(from.ID) > 0 { srcDigest = digest.Digest(from.ID) } else if len(from.Tag) > 0 { desc, err := repo.Tags(ctx).Get(ctx, from.Tag) if err != nil { - return nil, ManifestLocation{}, err + return from, nil, ManifestLocation{}, err } srcDigest = desc.Digest } else { - return nil, ManifestLocation{}, fmt.Errorf("no tag or digest specified") + return from, nil, ManifestLocation{}, fmt.Errorf("no tag or digest specified") } manifests, err := repo.Manifests(ctx) if err != nil { - return nil, ManifestLocation{}, err + return from, nil, ManifestLocation{}, err } - srcManifest, err := manifests.Get(ctx, srcDigest, PreferManifestList) + srcManifest, ref, err := manifests.(registryclient.ManifestWithLocationService).GetWithLocation(ctx, srcDigest, PreferManifestList) if err != nil { - return nil, ManifestLocation{}, err + return from, nil, ManifestLocation{}, err } + if srcManifest == nil { + return from, nil, ManifestLocation{}, fmt.Errorf("manifest for %s is nil", from.String()) + } originalSrcDigest := srcDigest - srcManifests, srcManifest, srcDigest, err := ProcessManifestList(ctx, srcDigest, srcManifest, manifests, from, filterFn, false) + srcManifests, srcManifest, srcDigest, err := ProcessManifestList(ctx, srcDigest, srcManifest, manifests, ref, filterFn, false) if err != nil { - return nil, ManifestLocation{}, err + return from, nil, ManifestLocation{}, err } if len(srcManifests) == 0 { - return nil, ManifestLocation{}, fmt.Errorf("filtered all images from manifest list") + return from, nil, ManifestLocation{}, fmt.Errorf("filtered all images from manifest list") } if srcDigest != originalSrcDigest { - return srcManifest, ManifestLocation{Manifest: srcDigest, ManifestList: originalSrcDigest}, nil + return from, srcManifest, ManifestLocation{Manifest: srcDigest, ManifestList: originalSrcDigest}, nil } - return srcManifest, ManifestLocation{Manifest: srcDigest}, nil + return ref, srcManifest, ManifestLocation{Manifest: srcDigest}, nil } // ManifestToImageConfig takes an image manifest and converts it into a structured object. diff --git a/pkg/cli/image/manifest/security_options.go b/pkg/cli/image/manifest/security_options.go new file mode 100644 index 0000000000..189ce92879 --- /dev/null +++ b/pkg/cli/image/manifest/security_options.go @@ -0,0 +1,129 @@ +package manifest + +import ( + "fmt" + "net/http" + + "github.com/spf13/pflag" + + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + kcmdutil "k8s.io/kubectl/pkg/cmd/util" + + operatorv1alpha1client "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1alpha1" + "github.com/openshift/library-go/pkg/image/registryclient" + "github.com/openshift/library-go/pkg/image/strategy" + "github.com/openshift/oc/pkg/cli/image/manifest/dockercredentials" +) + +type SecurityOptions struct { + RegistryConfig string + Insecure bool + SkipVerification bool + ICSPFile string + ICSPClientFn func() (operatorv1alpha1client.ImageContentSourcePolicyInterface, error) + LookupClusterICSP bool + FileDir string + + CachedContext *registryclient.Context +} + +func (o *SecurityOptions) Bind(flags *pflag.FlagSet) { + flags.StringVarP(&o.RegistryConfig, "registry-config", "a", o.RegistryConfig, "Path to your registry credentials (defaults to ~/.docker/config.json)") + flags.BoolVar(&o.Insecure, "insecure", o.Insecure, "Allow push and pull operations to registries to be made over HTTP") + flags.BoolVar(&o.SkipVerification, "skip-verification", o.SkipVerification, "Skip verifying the integrity of the retrieved content. This is not recommended, but may be necessary when importing images from older image registries. Only bypass verification if the registry is known to be trustworthy.") + flags.BoolVar(&o.LookupClusterICSP, "lookup-cluster-icsp", o.LookupClusterICSP, "default=true with 'oc adm release', default=false with 'oc image'. When explicitly set to true, look for alternative image sources from ImageContentSourcePolicy objects in cluster, honor the ordering of those sources, and fail if an ImageContentSourcePolicy is not found in cluster. Cannot be set to true with --icsp-file. Note: with implicit lookup, command will not error if unable to find any cluster ICSPs.") + flags.StringVar(&o.ICSPFile, "icsp-file", o.ICSPFile, "Path to an ImageContentSourcePolicy file. If set, data from this file will be used to set alternative image sources. Cannot be set together with --lookup-cluster-icsp=true.") +} + +// ReferentialHTTPClient returns an http.Client that is appropriate for accessing +// blobs referenced outside of the registry (due to the present of the URLs attribute +// in the manifest reference for a layer). +func (o *SecurityOptions) ReferentialHTTPClient() (*http.Client, error) { + regContext, err := o.Context() + if err != nil { + return nil, err + } + client := &http.Client{} + if o.Insecure { + client.Transport = regContext.InsecureTransport + } else { + client.Transport = regContext.Transport + } + return client, nil +} + +func (o *SecurityOptions) Complete(f kcmdutil.Factory) error { + if o.LookupClusterICSP && len(o.ICSPFile) > 0 { + return fmt.Errorf("cannot set both --lookup-cluster-icsp=true and --icsp-file") + } + if o.LookupClusterICSP { + o.ICSPClientFn = func() (operatorv1alpha1client.ImageContentSourcePolicyInterface, error) { + // If ImageContentSourceFile is given, only add ImageContentSource from file, don't search cluster ICSP + if len(o.ICSPFile) != 0 { + return nil, nil + } + restConfig, err := f.ToRESTConfig() + if err != nil { + // may or may not be connected to a cluster + // don't error if can't connect + klog.V(4).Infof("did not connect to an OpenShift 4.x server, will not lookup ImageContentSourcePolicies: %v", err) + return nil, nil + } + icspClient, err := operatorv1alpha1client.NewForConfig(restConfig) + if err != nil { + // may or may not be connected to a cluster + // don't error if can't connect + klog.V(4).Infof("did not connect to an OpenShift 4.x server, will not lookup ImageContentSourcePolicies: %v", err) + return nil, nil + } + return icspClient.ImageContentSourcePolicies(), nil + } + } + return nil +} + +func (o *SecurityOptions) Context() (*registryclient.Context, error) { + if o.CachedContext != nil { + return o.CachedContext, nil + } + context, err := o.NewContext() + if err == nil { + o.CachedContext = context + o.CachedContext.Retries = 3 + } + return context, err +} + +func (o *SecurityOptions) NewContext() (*registryclient.Context, error) { + rt, err := rest.TransportFor(&rest.Config{}) + if err != nil { + return nil, err + } + insecureRT, err := rest.TransportFor(&rest.Config{TLSClientConfig: rest.TLSClientConfig{Insecure: true}}) + if err != nil { + return nil, err + } + creds := dockercredentials.NewLocal() + if len(o.RegistryConfig) > 0 { + creds, err = dockercredentials.NewFromFile(o.RegistryConfig) + if err != nil { + return nil, fmt.Errorf("unable to load --registry-config: %v", err) + } + } + + var regContext *registryclient.Context + var icspClient operatorv1alpha1client.ImageContentSourcePolicyInterface + if o.LookupClusterICSP { + icspClient, err = o.ICSPClientFn() + if err != nil { + return nil, err + } + } + regContext = registryclient.NewContext(rt, insecureRT).WithCredentials(creds) + if len(o.ICSPFile) > 0 || o.LookupClusterICSP { + regContext = regContext.WithAlternateBlobSourceStrategy(strategy.NewSimpleLookupICSPStrategy(o.ICSPFile, icspClient)) + } + regContext.DisableDigestVerification = o.SkipVerification + return regContext, nil +} diff --git a/pkg/cli/image/mirror/mirror.go b/pkg/cli/image/mirror/mirror.go index 965c3f3eab..2622165569 100644 --- a/pkg/cli/image/mirror/mirror.go +++ b/pkg/cli/image/mirror/mirror.go @@ -151,7 +151,7 @@ func NewMirrorImageOptions(streams genericclioptions.IOStreams) *MirrorImageOpti } // NewCommandMirrorImage copies images from one location to another. -func NewCmdMirrorImage(streams genericclioptions.IOStreams) *cobra.Command { +func NewCmdMirrorImage(f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewMirrorImageOptions(streams) cmd := &cobra.Command{ @@ -160,7 +160,7 @@ func NewCmdMirrorImage(streams genericclioptions.IOStreams) *cobra.Command { Long: mirrorDesc, Example: mirrorExample, Run: func(c *cobra.Command, args []string) { - kcmdutil.CheckErr(o.Complete(c, args)) + kcmdutil.CheckErr(o.Complete(f, c, args)) kcmdutil.CheckErr(o.Validate()) kcmdutil.CheckErr(o.Run()) }, @@ -187,7 +187,7 @@ func NewCmdMirrorImage(streams genericclioptions.IOStreams) *cobra.Command { return cmd } -func (o *MirrorImageOptions) Complete(cmd *cobra.Command, args []string) error { +func (o *MirrorImageOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { if o.KeepManifestList && len(o.FilterOptions.FilterByOS) == 0 { o.FilterOptions.FilterByOS = ".*" } @@ -196,13 +196,12 @@ func (o *MirrorImageOptions) Complete(cmd *cobra.Command, args []string) error { return err } - if o.FilterOptions.IsWildcardFilter() { - o.KeepManifestList = true + if err := o.SecurityOptions.Complete(f); err != nil { + return err } - registryContext, err := o.SecurityOptions.Context() - if err != nil { - return err + if o.FilterOptions.IsWildcardFilter() { + o.KeepManifestList = true } dir := o.FileDir @@ -210,6 +209,11 @@ func (o *MirrorImageOptions) Complete(cmd *cobra.Command, args []string) error { dir = o.FromFileDir } + registryContext, err := o.SecurityOptions.Context() + if err != nil { + return err + } + opts := &imagesource.Options{ FileDir: dir, Insecure: o.SecurityOptions.Insecure, @@ -243,7 +247,7 @@ func (o *MirrorImageOptions) Complete(cmd *cobra.Command, args []string) error { return nil } -func (o *MirrorImageOptions) Repository(ctx context.Context, context *registryclient.Context, ref imagesource.TypedImageReference, source bool) (distribution.Repository, error) { +func (o *MirrorImageOptions) Repository(ctx context.Context, regContext *registryclient.Context, ref imagesource.TypedImageReference, source bool) (distribution.Repository, error) { dir := o.FileDir if len(o.FromFileDir) > 0 && source { dir = o.FromFileDir @@ -253,7 +257,7 @@ func (o *MirrorImageOptions) Repository(ctx context.Context, context *registrycl FileDir: dir, Insecure: o.SecurityOptions.Insecure, AttemptS3BucketCopy: o.AttemptS3BucketCopy, - RegistryContext: context, + RegistryContext: regContext, } return opts.Repository(ctx, ref) } @@ -579,7 +583,8 @@ func (o *MirrorImageOptions) plan() (*plan, error) { case src.ref.EqualRegistry(dst.ref) && canonicalFrom.String() == canonicalTo.String(): // if the source and destination repos are the same, we don't need to copy layers unless forced default: - if _, err := toManifests.Get(ctx, srcDigest); err != nil { + m, err := toManifests.Get(ctx, srcDigest) + if err != nil || m == nil { mustCopyLayers = true blobPlan.AlreadyExists(distribution.Descriptor{Digest: srcDigest}) } else { @@ -647,8 +652,8 @@ func (o *MirrorImageOptions) plan() (*plan, error) { func copyBlob(ctx context.Context, plan *workPlan, c *repositoryBlobCopy, blob distribution.Descriptor, referentialClient *http.Client, force, skipMount bool, errOut io.Writer) error { // if we aren't forcing upload, check to see if the blob aleady exists if !force { - _, err := c.to.Stat(ctx, blob.Digest) - if err == nil { + b, err := c.to.Stat(ctx, blob.Digest) + if err == nil && b.Size > 0 { // blob exists, skip klog.V(5).Infof("Server reports blob exists %#v", blob) c.parent.parent.AssociateBlob(c.parent.name, blob) diff --git a/pkg/cli/registry/login/login.go b/pkg/cli/registry/login/login.go index 1e5b6acfc5..c2320dec93 100644 --- a/pkg/cli/registry/login/login.go +++ b/pkg/cli/registry/login/login.go @@ -290,7 +290,7 @@ func (o *LoginOptions) Run() error { return err } c := registryclient.NewContext(http.DefaultTransport, insecureRT).WithCredentials(creds) - if _, err := c.Repository(ctx, url, "does_not_exist", o.Insecure); err != nil { + if _, err := c.RepositoryForRef(ctx, reference.DockerImageReference{Registry: url.Host, Namespace: "does_not_exist"}, o.Insecure); err != nil { return fmt.Errorf("unable to check your credentials - pass --skip-check to bypass this error: %v", err) } } diff --git a/vendor/github.com/openshift/library-go/pkg/config/client/transport.go b/vendor/github.com/openshift/library-go/pkg/config/client/transport.go index 65980c9afe..7f2534fdf1 100644 --- a/vendor/github.com/openshift/library-go/pkg/config/client/transport.go +++ b/vendor/github.com/openshift/library-go/pkg/config/client/transport.go @@ -52,14 +52,40 @@ type preferredHostRT struct { preferredHostFn func() string } -func (t *preferredHostRT) RoundTrip(r *http.Request) (*http.Response, error) { - preferredHost := t.preferredHostFn() +func (rt *preferredHostRT) RoundTrip(r *http.Request) (*http.Response, error) { + preferredHost := rt.preferredHostFn() if len(preferredHost) == 0 { - return t.baseRT.RoundTrip(r) + return rt.baseRT.RoundTrip(r) } r.Host = preferredHost r.URL.Host = preferredHost - return t.baseRT.RoundTrip(r) + return rt.baseRT.RoundTrip(r) +} + +// CancelRequest exists to facilitate cancellation. +// +// In general there are at least three ways of cancelling a request by an HTTP client: +// 1. Transport.CancelRequest (depreciated) +// 2. Request.Cancel +// 3. Request.Context (preferred) +// +// While using client-go callers can specify a timeout value that gets passed directly to an http.Client. +// The HTTP client cancels requests to the underlying Transport as if the Request's Context ended. +// For compatibility, the Client will also use the deprecated CancelRequest method on Transport if found. +// New RoundTripper implementations should use the Request's Context for cancellation instead of implementing CancelRequest. +// +// Because this wrapper might be the first or might be actually wrapped with already existing wrappers that already implement CancelRequest we need to simply conform. +// +// See for more details: +// https://github.com/kubernetes/kubernetes/blob/442a69c3bdf6fe8e525b05887e57d89db1e2f3a5/staging/src/k8s.io/client-go/transport/transport.go#L257 +// https://github.com/kubernetes/kubernetes/blob/e29c568c4a9cd45d15665345aa015e21bcff52dd/staging/src/k8s.io/client-go/rest/config.go#L328 +// https://github.com/kubernetes/kubernetes/blob/3b2746c9ea9e0fa247b01dca27634e509b385eda/staging/src/k8s.io/client-go/transport/round_trippers.go#L302 +func (rt *preferredHostRT) CancelRequest(req *http.Request) { + type canceler interface{ CancelRequest(*http.Request) } + + if rtCanceller, ok := rt.baseRT.(canceler); ok { + rtCanceller.CancelRequest(req) + } } diff --git a/vendor/github.com/openshift/library-go/pkg/image/registryclient/client.go b/vendor/github.com/openshift/library-go/pkg/image/registryclient/client.go index 1689a5cb49..fde78dd482 100644 --- a/vendor/github.com/openshift/library-go/pkg/image/registryclient/client.go +++ b/vendor/github.com/openshift/library-go/pkg/image/registryclient/client.go @@ -26,6 +26,8 @@ import ( "github.com/docker/distribution/registry/client/auth/challenge" "github.com/docker/distribution/registry/client/transport" "github.com/opencontainers/go-digest" + + imagereference "github.com/openshift/library-go/pkg/image/reference" ) // RepositoryRetriever fetches a Docker distribution.Repository. @@ -80,6 +82,7 @@ type Context struct { Credentials auth.CredentialStore RequestModifiers []transport.RequestModifier Limiter *rate.Limiter + Alternates AlternateBlobSourceStrategy DisableDigestVerification bool @@ -138,6 +141,11 @@ func (c *Context) WithCredentials(credentials auth.CredentialStore) *Context { return c } +func (c *Context) WithAlternateBlobSourceStrategy(alternateStrategy AlternateBlobSourceStrategy) *Context { + c.Alternates = alternateStrategy + return c +} + // Reset clears any cached repository info for this context. func (c *Context) Reset() { c.lock.Lock() @@ -200,18 +208,67 @@ func (c *Context) Ping(ctx context.Context, registry *url.URL, insecure bool) (h return t, &src, nil } -func (c *Context) Repository(ctx context.Context, registry *url.URL, repoName string, insecure bool) (distribution.Repository, error) { +// RepositoryForRef returns a distribution.Repository against the provided image reference. If insecure +// is true, HTTP connections are allowed and HTTPS certificate verification errors will be ignored. The returned +// Repository instance is threadsafe but the ManifestService, TagService, or BlobService are not. +func (c *Context) RepositoryForRef(ctx context.Context, ref imagereference.DockerImageReference, insecure bool) (distribution.Repository, error) { + return c.connectToRegistry(ctx, repositoryLocator{ref: ref}, insecure) +} + +// Repository returns a distribution.Repository against the provided registry and repository name. If insecure +// is true, HTTP connections are allowed and HTTPS certificate verification errors will be ignored. The returned +// Repository instance is threadsafe but the ManifestService, TagService, or BlobService are not. Note - the caller +// is responsible for providing a valid registry url for docker.io - use RepositoryForRef() to avoid that. +func (c *Context) Repository(ctx context.Context, registry *url.URL, repoName string, insecure bool) (RepositoryWithLocation, error) { named, err := reference.WithName(repoName) if err != nil { return nil, err } + ref, err := imagereference.Parse(repoName) + if err != nil { + return nil, err + } + ref.Registry = registry.Host + locator := repositoryLocator{ + named: named, + ref: ref, + url: registry, + } + return &blobMirroredRepository{ + locator: locator, + strategy: c.Alternates, + retriever: c, + }, nil +} + +// connectToRegistry is private and returns a non-wrapped, non-mirrorable repository. +func (c *Context) connectToRegistry(ctx context.Context, locator repositoryLocator, insecure bool) (RepositoryWithLocation, error) { + var named reference.Named = locator.named + var registryURL *url.URL = locator.url + var path string + + // ensure the values needed from the locator are defaulted + if named == nil { + path = locator.ref.RepositoryName() + var err error + named, err = reference.WithName(path) + if err != nil { + return nil, err + } + } else { + path = reference.Path(named) + } + if registryURL == nil { + registryURL = locator.ref.RegistryURL() + } - rt, src, err := c.Ping(ctx, registry, insecure) + // attempt to connect to the registry to get auth instructions + rt, src, err := c.Ping(ctx, registryURL, insecure) if err != nil { return nil, err } - rt = c.repositoryTransport(rt, src, repoName) + rt = c.repositoryTransport(rt, src, path) repo, err := registryclient.NewRepository(named, src.String(), rt) if err != nil { @@ -224,7 +281,7 @@ func (c *Context) Repository(ctx context.Context, registry *url.URL, repoName st if limiter == nil { limiter = rate.NewLimiter(rate.Limit(5), 5) } - return NewLimitedRetryRepository(repo, c.Retries, limiter), nil + return NewLimitedRetryRepository(locator.ref, repo, c.Retries, limiter), nil } func (c *Context) ping(registry url.URL, insecure bool, transport http.RoundTripper) (*url.URL, error) { @@ -359,6 +416,7 @@ var nowFn = time.Now type retryRepository struct { distribution.Repository + ref imagereference.DockerImageReference limiter *rate.Limiter retries int sleepFn func(time.Duration) @@ -366,16 +424,21 @@ type retryRepository struct { // NewLimitedRetryRepository wraps a distribution.Repository with helpers that will retry temporary failures // over a limited time window and duration, and also obeys a rate limit. -func NewLimitedRetryRepository(repo distribution.Repository, retries int, limiter *rate.Limiter) distribution.Repository { +func NewLimitedRetryRepository(ref imagereference.DockerImageReference, repo distribution.Repository, retries int, limiter *rate.Limiter) RepositoryWithLocation { return &retryRepository{ Repository: repo, + ref: ref, limiter: limiter, retries: retries, sleepFn: time.Sleep, } } +func (r *retryRepository) Ref() imagereference.DockerImageReference { + return r.ref +} + // isTemporaryHTTPError returns true if the error indicates a temporary or partial HTTP failure func isTemporaryHTTPError(err error) (time.Duration, bool) { if err == nil { diff --git a/vendor/github.com/openshift/library-go/pkg/image/registryclient/client_mirrored.go b/vendor/github.com/openshift/library-go/pkg/image/registryclient/client_mirrored.go new file mode 100644 index 0000000000..8132b2fcc2 --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/image/registryclient/client_mirrored.go @@ -0,0 +1,521 @@ +package registryclient + +import ( + "fmt" + "net/http" + "net/url" + "sync" + + "github.com/docker/distribution" + "github.com/opencontainers/go-digest" + "github.com/openshift/library-go/pkg/image/reference" + "golang.org/x/net/context" + "k8s.io/klog/v2" + + distributionreference "github.com/docker/distribution/reference" +) + +// AlternateBlobSourceStrategy is consulted when a repository cannot be reached to find alternate +// repositories that may be able to serve a given content-addressed blob. The strategy is consulted +// at most twice - once before any request is made to a given repository. If FirstRequest() returns a +// list of alternates, OnFailure is not invoked. +type AlternateBlobSourceStrategy interface { + // FirstRequest returns the set of locations that should be searched in a preferred order. If locator + // is not included in the response it will not be searched. If alternateRepositories is an empty list + // no lookup will be performed and requests will exit with an error. If alternateRepositories is nil + // and err is nil, OnFailure will be invoked if the first request fails. + FirstRequest(ctx context.Context, locator reference.DockerImageReference) (alternateRepositories []reference.DockerImageReference, err error) + // OnFailure is invoked if FirstRequest returned no error and a nil list of locations if and only if + // an API call fails on the specified request. The result of alternateRepositories is cached for + // subsequent calls to that repository. + OnFailure(ctx context.Context, locator reference.DockerImageReference) (alternateRepositories []reference.DockerImageReference, err error) +} + +// ManifestWithLocationService extends the ManifestService to allow clients to retrieve a manifest and +// get the location of the mirrored manifest. Not all ManifestServices returned from a Repository will +// support this interface and it must be conditional. +type ManifestWithLocationService interface { + distribution.ManifestService + + // GetWithLocation returns the registry URL the provided manifest digest was retrieved from which may be Repository.Named(), + // or one of the blob mirrors if alternate location for blob sources was provided. It returns an error if the digest could not be + // located - if an error is returned the source reference (Repository.Named()) will be set. + GetWithLocation(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, reference.DockerImageReference, error) +} + +// RepositoryWithLocation extends the Repository and allows clients to know which repository registry this talks to +// as primary (as a complement to Named() which does not include the URL). +type RepositoryWithLocation interface { + distribution.Repository + + // Ref returns the DockerImageReference representing this repository. + Ref() reference.DockerImageReference +} + +// blobMirroredRepoRetriever allows a caller to retrieve a distribution.Repository. It may perform +// requests to authorize the client and will return an error if it fails. +type blobMirroredRepoRetriever interface { + connectToRegistry(context.Context, repositoryLocator, bool) (RepositoryWithLocation, error) +} + +// repositoryLocator caches the components necessary to connect to a single image repository. +type repositoryLocator struct { + ref reference.DockerImageReference + // url may specify a default protocol (http) instead of (https), but is otherwise calculated + // by taking ref.Registry and applying it to url.Host + url *url.URL + // named is the image repository path on the server (namespace and name in ref terms) and is + // required for the distribution registryclient. + named distributionreference.Named +} + +// blobMirroredRepository provides failover lookup behavior for blobs in a given repository on +// errors by delegating to the provided strategy for the first request or when a failure occurs. +// The strategy is expected to return a set of alternate locations to consume content from, +// which may not include the original source. Only requests made for content addressable blobs +// may be consulted in this fashion (anything via digest) - everything else must use source(). +type blobMirroredRepository struct { + locator repositoryLocator + insecure bool + + strategy AlternateBlobSourceStrategy + retriever blobMirroredRepoRetriever + + lock sync.Mutex + order []reference.DockerImageReference + repos map[reference.DockerImageReference]RepositoryWithLocation +} + +// Named returns the name of the repository. +func (r *blobMirroredRepository) Named() distributionreference.Named { + return r.locator.named +} + +// Named returns the name of the repository. +func (r *blobMirroredRepository) Ref() reference.DockerImageReference { + return r.locator.ref +} + +// Manifests wraps the manifest service in a blobMirroredManifest for shared retries. +func (r *blobMirroredRepository) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) { + return &blobMirroredManifest{repo: r, options: options}, nil +} + +// Blobs wraps the blob service in a blobMirroredBlobstore for shared retries. +func (r *blobMirroredRepository) Blobs(ctx context.Context) distribution.BlobStore { + return blobMirroredBlobstore{repo: r} +} + +// Tags lists the tags under the named repository. +func (r *blobMirroredRepository) Tags(ctx context.Context) distribution.TagService { + return blobMirroredTags{repo: r} +} + +var ( + errNoValidAlternates = fmt.Errorf("no valid alterative sources for this content located") + errNoValidSource = fmt.Errorf("no source repository defined for accessing the repository") +) + +// initialRepos returns a list of locations to attempt to access, a boolean indicating that alternates +// were suggested, or an error. +func (r *blobMirroredRepository) initialRepos(ctx context.Context) ([]reference.DockerImageReference, bool, error) { + if r.strategy == nil { + return []reference.DockerImageReference{r.locator.ref}, false, nil + } + + if r.order != nil { + return r.order, true, nil + } + alternates, err := r.strategy.FirstRequest(ctx, r.locator.ref) + if err != nil { + return nil, false, err + } + if len(alternates) == 0 { + alternates, err = r.errorRepos(ctx) + } + + r.order = alternates + return r.order, true, nil +} + +// errorRepos returns a list of alternate registries to search for the provided content. +func (r *blobMirroredRepository) errorRepos(ctx context.Context) ([]reference.DockerImageReference, error) { + if r.strategy == nil { + return nil, nil + } + + // TODO: potentially filter certain types of errors, maybe even per method type, if we ever + // retry non-idempotent operations + r.lock.Lock() + defer r.lock.Unlock() + if r.order != nil { + return nil, nil + } + alternates, err := r.strategy.OnFailure(ctx, r.locator.ref) + if err != nil { + return nil, err + } + r.order = alternates + return r.order, nil +} + +// attemptRepos will invoke fn on all repos until fn returns no error. fn is expected to be idempotent. +func (r *blobMirroredRepository) attemptRepos(ctx context.Context, repos []reference.DockerImageReference, fn func(r RepositoryWithLocation) error) error { + var firstErr error + for _, ref := range repos { + klog.V(5).Infof("Attempting to connect to %s", ref) + repo, err := r.connect(ctx, ref) + if err != nil { + if firstErr == nil { + firstErr = err + } + continue + } + if err := fn(repo); err != nil { + if firstErr == nil { + firstErr = err + } + continue + } + return nil + } + return firstErr +} + +// attemptFirstConnectedRepo will invoke fn on the first repo that successfully connects. +func (r *blobMirroredRepository) attemptFirstConnectedRepo(ctx context.Context, repos []reference.DockerImageReference, fn func(r RepositoryWithLocation) error) error { + var firstErr error + for _, ref := range repos { + klog.V(5).Infof("Attempting to connect to %s", ref) + repo, err := r.connect(ctx, ref) + if err != nil { + if firstErr == nil { + firstErr = err + } + continue + } + return fn(repo) + } + return firstErr +} + +// alternates accesses the set of repositories that may be valid alternatives for accessing content +func (r *blobMirroredRepository) alternates(ctx context.Context, fn func(r RepositoryWithLocation) error) error { + repos, loaded, err := r.initialRepos(ctx) + if err != nil { + return err + } + if attemptErr := r.attemptRepos(ctx, repos, fn); attemptErr != nil { + if loaded { + return attemptErr + } + alternates, err := r.errorRepos(ctx) + if err != nil { + return err + } + if alternateErr := r.attemptRepos(ctx, alternates, fn); alternateErr != nil { + return attemptErr + } + } + return nil +} + +// firstConnectedAlternate invokes fn on the first alternate that can be connected to. Use when the +// function can only be invoked once (such as a method with side effects, like ServeBlob which writes +// to the response). +func (r *blobMirroredRepository) firstConnectedAlternate(ctx context.Context, fn func(r RepositoryWithLocation) error) error { + repos, loaded, err := r.initialRepos(ctx) + if err != nil { + return err + } + if len(repos) == 0 { + return errNoValidAlternates + } + if attemptErr := r.attemptFirstConnectedRepo(ctx, repos, fn); attemptErr != nil { + if loaded { + return attemptErr + } + alternates, err := r.errorRepos(ctx) + if err != nil { + return err + } + if alternateErr := r.attemptFirstConnectedRepo(ctx, alternates, fn); alternateErr != nil { + return attemptErr + } + } + return nil +} + +// source connects to the original repository or returns an error. It will always use the same value +// of insecure as the original repository. Use when the request should only go to the initial repo. +func (r *blobMirroredRepository) source(ctx context.Context, fn func(r distribution.Repository) error) error { + repo, err := r.connect(ctx, r.locator.ref) + if err != nil { + return err + } + return fn(repo) +} + +// connect reuses or creates a connection to the provided reference, returning a repository instance +// or an error. This method expects that the connection only talks to the provided registry. +func (r *blobMirroredRepository) connect(ctx context.Context, ref reference.DockerImageReference) (RepositoryWithLocation, error) { + r.lock.Lock() + defer r.lock.Unlock() + + repo, ok := r.repos[ref] + if ok { + return repo, nil + } + locator := repositoryLocator{ + ref: ref, + } + repo, err := r.retriever.connectToRegistry(ctx, locator, ref != r.locator.ref || r.insecure) + if err != nil { + return nil, err + } + if r.repos == nil { + r.repos = make(map[reference.DockerImageReference]RepositoryWithLocation) + } + r.repos[ref] = repo + return repo, nil +} + +// blobMirroredManifest will sequentially retry manifest operations on a set of repositories determined +// by the repository list, caching manifest services locally as needed (manifest service is assumed +// to have local state and does so in the registry client). The individual manifest service is not +// thread safe, but methods on this interface are thread safe. +type blobMirroredManifest struct { + repo *blobMirroredRepository + options []distribution.ManifestServiceOption + + lock sync.Mutex + cache map[distribution.Repository]distribution.ManifestService +} + +var _ distribution.ManifestService = &blobMirroredManifest{} +var _ ManifestWithLocationService = &blobMirroredManifest{} + +// init retrieves or caches a manifets service for the provided repository, since each manifest +// service has local state. +func (f *blobMirroredManifest) init(ctx context.Context, r distribution.Repository) (distribution.ManifestService, error) { + f.lock.Lock() + defer f.lock.Unlock() + + ms := f.cache[r] + if ms != nil { + return ms, nil + } + ms, err := r.Manifests(ctx, f.options...) + if err != nil { + return nil, err + } + if f.cache == nil { + f.cache = make(map[distribution.Repository]distribution.ManifestService) + } + f.cache[r] = ms + return ms, nil +} + +// alternates invokes fn once per alternate repo until fn returns without error. +func (f *blobMirroredManifest) alternates(ctx context.Context, fn func(m distribution.ManifestService, repo RepositoryWithLocation) error) error { + return f.repo.alternates(ctx, func(repo RepositoryWithLocation) error { + ms, err := f.init(ctx, repo) + if err != nil { + return err + } + return fn(ms, repo) + }) +} + +// source invokes fn against the primary location. +func (f *blobMirroredManifest) source(ctx context.Context, fn func(r distribution.ManifestService) error) error { + return f.repo.source(ctx, func(r distribution.Repository) error { + ms, err := f.init(ctx, r) + if err != nil { + return err + } + return fn(ms) + }) +} + +func (f *blobMirroredManifest) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { + var dgst digest.Digest + err := f.source(ctx, func(r distribution.ManifestService) error { + var err error + dgst, err = r.Put(ctx, manifest, options...) + return err + }) + return dgst, err +} + +func (f *blobMirroredManifest) Delete(ctx context.Context, dgst digest.Digest) error { + return f.source(ctx, func(r distribution.ManifestService) error { + return r.Delete(ctx, dgst) + }) +} + +func (f *blobMirroredManifest) Exists(ctx context.Context, dgst digest.Digest) (bool, error) { + var ok bool + err := f.alternates(ctx, func(m distribution.ManifestService, repo RepositoryWithLocation) error { + var err error + ok, err = m.Exists(ctx, dgst) + return err + }) + return ok, err +} + +func (f *blobMirroredManifest) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { + var manifest distribution.Manifest + err := f.alternates(ctx, func(m distribution.ManifestService, repo RepositoryWithLocation) error { + var err error + manifest, err = m.Get(ctx, dgst, options...) + klog.V(5).Infof("get manifest for %s served from %#v: %v", dgst, m, err) + return err + }) + return manifest, err +} + +func (f *blobMirroredManifest) GetWithLocation(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, reference.DockerImageReference, error) { + var manifest distribution.Manifest + var ref = f.repo.locator.ref + err := f.alternates(ctx, func(m distribution.ManifestService, repo RepositoryWithLocation) error { + var err error + manifest, err = m.Get(ctx, dgst, options...) + klog.V(5).Infof("get manifest for %s served from %#v: %v", dgst, m, err) + if err == nil { + ref = repo.Ref() + } + return err + }) + return manifest, ref, err +} + +// blobMirroredBlobstore wraps the blob store and invokes retries on the repo. +type blobMirroredBlobstore struct { + repo *blobMirroredRepository +} + +var _ distribution.BlobService = blobMirroredBlobstore{} + +func (f blobMirroredBlobstore) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { + var data []byte + err := f.repo.alternates(ctx, func(r RepositoryWithLocation) error { + var err error + data, err = r.Blobs(ctx).Get(ctx, dgst) + klog.V(5).Infof("get for %s served from %s: %v", dgst, r.Named(), err) + return err + }) + return data, err +} + +func (f blobMirroredBlobstore) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + var desc distribution.Descriptor + err := f.repo.alternates(ctx, func(r RepositoryWithLocation) error { + var err error + desc, err = r.Blobs(ctx).Stat(ctx, dgst) + return err + }) + return desc, err +} + +func (f blobMirroredBlobstore) ServeBlob(ctx context.Context, w http.ResponseWriter, req *http.Request, dgst digest.Digest) error { + err := f.repo.firstConnectedAlternate(ctx, func(r RepositoryWithLocation) error { + return r.Blobs(ctx).ServeBlob(ctx, w, req, dgst) + }) + return err +} + +func (f blobMirroredBlobstore) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) { + var rsc distribution.ReadSeekCloser + err := f.repo.alternates(ctx, func(r RepositoryWithLocation) error { + var err error + rsc, err = r.Blobs(ctx).Open(ctx, dgst) + return err + }) + return rsc, err +} + +func (f blobMirroredBlobstore) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) { + var bw distribution.BlobWriter + err := f.repo.source(ctx, func(r distribution.Repository) error { + var err error + bw, err = r.Blobs(ctx).Create(ctx, options...) + return err + }) + return bw, err +} + +func (f blobMirroredBlobstore) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) { + var desc distribution.Descriptor + err := f.repo.source(ctx, func(r distribution.Repository) error { + var err error + desc, err = r.Blobs(ctx).Put(ctx, mediaType, p) + return err + }) + return desc, err +} + +func (f blobMirroredBlobstore) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) { + var bw distribution.BlobWriter + err := f.repo.source(ctx, func(r distribution.Repository) error { + var err error + bw, err = r.Blobs(ctx).Resume(ctx, id) + return err + }) + return bw, err +} + +func (f blobMirroredBlobstore) Delete(ctx context.Context, dgst digest.Digest) error { + return f.repo.source(ctx, func(r distribution.Repository) error { + return r.Blobs(ctx).Delete(ctx, dgst) + }) +} + +// blobMirroredTags lazily accesses the source repository +type blobMirroredTags struct { + repo *blobMirroredRepository +} + +var _ distribution.TagService = blobMirroredTags{} + +func (f blobMirroredTags) Get(ctx context.Context, tag string) (distribution.Descriptor, error) { + var desc distribution.Descriptor + err := f.repo.source(ctx, func(r distribution.Repository) error { + var err error + desc, err = r.Tags(ctx).Get(ctx, tag) + return err + }) + return desc, err +} + +func (f blobMirroredTags) All(ctx context.Context) ([]string, error) { + var tags []string + err := f.repo.source(ctx, func(r distribution.Repository) error { + var err error + tags, err = r.Tags(ctx).All(ctx) + return err + }) + return tags, err +} + +func (f blobMirroredTags) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) { + var tags []string + err := f.repo.source(ctx, func(r distribution.Repository) error { + var err error + tags, err = r.Tags(ctx).Lookup(ctx, digest) + return err + }) + return tags, err +} + +func (f blobMirroredTags) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error { + return f.repo.source(ctx, func(r distribution.Repository) error { + return r.Tags(ctx).Tag(ctx, tag, desc) + }) +} + +func (f blobMirroredTags) Untag(ctx context.Context, tag string) error { + return f.repo.source(ctx, func(r distribution.Repository) error { + return r.Tags(ctx).Untag(ctx, tag) + }) +} diff --git a/vendor/github.com/openshift/library-go/pkg/image/strategy/simplelookup.go b/vendor/github.com/openshift/library-go/pkg/image/strategy/simplelookup.go new file mode 100644 index 0000000000..b62ba4592a --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/image/strategy/simplelookup.go @@ -0,0 +1,159 @@ +package strategy + +import ( + "context" + "fmt" + "io/ioutil" + "sync" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" + + operatorv1alpha1 "github.com/openshift/api/operator/v1alpha1" + operatorv1alpha1scheme "github.com/openshift/client-go/operator/clientset/versioned/scheme" + operatorv1alpha1client "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1alpha1" + reference "github.com/openshift/library-go/pkg/image/reference" + "github.com/openshift/library-go/pkg/image/registryclient" +) + +// simpleLookupICSP holds ImageContentSourcePolicy variables to look up image sources +// satisfies *Context AlternativeBlobSourceStrategy interface +type simpleLookupICSP struct { + lock sync.Mutex + + alternates []reference.DockerImageReference + icspFile string + icspClient operatorv1alpha1client.ImageContentSourcePolicyInterface +} + +func NewSimpleLookupICSPStrategy(file string, client operatorv1alpha1client.ImageContentSourcePolicyInterface) registryclient.AlternateBlobSourceStrategy { + return &simpleLookupICSP{ + icspFile: file, + icspClient: client, + } +} + +func (s *simpleLookupICSP) FirstRequest(ctx context.Context, locator reference.DockerImageReference) (alternateRepositories []reference.DockerImageReference, err error) { + return nil, nil +} + +// OnFailure returns a list of possible image references for the locator image reference, gathered from image content source policies, or an error +func (s *simpleLookupICSP) OnFailure(ctx context.Context, locator reference.DockerImageReference) (alternateRepositories []reference.DockerImageReference, err error) { + s.lock.Lock() + defer s.lock.Unlock() + if err := s.resolve(ctx, locator); err != nil { + return nil, err + } + if len(s.alternates) == 0 { + return nil, fmt.Errorf("no alternative image references found for image: %s", locator.String()) + } + return s.alternates, nil +} + +// addICSPsFromCluster will lookup ImageContentSourcePolicy resources in cluster. +func (s *simpleLookupICSP) addICSPsFromCluster(ctx context.Context) ([]operatorv1alpha1.ImageContentSourcePolicy, error) { + if s.icspClient == nil { + return nil, fmt.Errorf("no client to access ImageContentSourcePolicies in cluster") + } + icsps, err := s.icspClient.List(ctx, metav1.ListOptions{}) + if err != nil { + // may or may not have access to ICSPs in cluster + // don't error if can't access ICSPs + return nil, fmt.Errorf("did not access any ImageContentSourcePolicies in cluster: %v", err) + } + if len(icsps.Items) == 0 { + return nil, fmt.Errorf("no ImageContentSourcePolicies found in cluster") + } + return icsps.Items, nil +} + +// addICSPsFromFile appends to list of alternative image sources from ICSP file +// returns error if no icsp object decoded from file data +func (s *simpleLookupICSP) addICSPsFromFile() ([]operatorv1alpha1.ImageContentSourcePolicy, error) { + icspData, err := ioutil.ReadFile(s.icspFile) + if err != nil { + return nil, fmt.Errorf("unable to read ImageContentSourceFile %s: %v", s.icspFile, err) + } + if len(icspData) == 0 { + return nil, fmt.Errorf("no data found in ImageContentSourceFile %s", s.icspFile) + } + icspObj, err := runtime.Decode(operatorv1alpha1scheme.Codecs.UniversalDeserializer(), icspData) + if err != nil { + return nil, fmt.Errorf("error decoding ImageContentSourcePolicy from %s: %v", s.icspFile, err) + } + icsp, ok := icspObj.(*operatorv1alpha1.ImageContentSourcePolicy) + if !ok { + return nil, fmt.Errorf("could not decode ImageContentSourcePolicy from %s", s.icspFile) + } + return []operatorv1alpha1.ImageContentSourcePolicy{*icsp}, nil +} + +// alternativeImageSources returns unique list of DockerImageReference objects from list of ImageContentSourcePolicy objects +func (s *simpleLookupICSP) alternativeImageSources(imageRef reference.DockerImageReference, icspList []operatorv1alpha1.ImageContentSourcePolicy) ([]reference.DockerImageReference, error) { + var imageSources []reference.DockerImageReference + // make sure at least 1 imagesource + // ie, make sure the image passed is included in image sources + // this is so the user-given image ref will be tried + imageSources = append(imageSources, imageRef.AsRepository()) + klog.V(2).Infof("%v ImageReference added to potential ImageSourcePrefixes from ImageContentSourcePolicy", imageRef.AsRepository()) + for _, icsp := range icspList { + repoDigestMirrors := icsp.Spec.RepositoryDigestMirrors + for _, rdm := range repoDigestMirrors { + var err error + rdmSourceRef, err := reference.Parse(rdm.Source) + if err != nil { + return nil, err + } + if imageRef.AsRepository() != rdmSourceRef.AsRepository() { + continue + } + klog.V(2).Infof("%v RepositoryDigestMirrors source matches given image", imageRef.AsRepository()) + for _, m := range rdm.Mirrors { + mRef, err := reference.Parse(m) + if err != nil { + return nil, err + } + imageSources = append(imageSources, mRef) + klog.V(2).Infof("%v RepositoryDigestMirrors mirror added to potential ImageSourcePrefixes from ImageContentSourcePolicy", m) + } + } + } + uniqueMirrors := make([]reference.DockerImageReference, 0, len(imageSources)) + uniqueMap := make(map[reference.DockerImageReference]bool) + for _, imageSourceMirror := range imageSources { + if _, ok := uniqueMap[imageSourceMirror]; !ok { + uniqueMap[imageSourceMirror] = true + uniqueMirrors = append(uniqueMirrors, imageSourceMirror) + } + } + klog.V(2).Infof("Found sources: %v for image: %v", uniqueMirrors, imageRef) + return uniqueMirrors, nil +} + +// resolve gathers possible image sources for a given image +// gathered from ImageContentSourcePolicy objects and user-passed image. +// Will lookup from cluster or from ImageContentSourcePolicy file passed from user. +// Image reference of user-given image may be different from original in case of mirrored images. +func (s *simpleLookupICSP) resolve(ctx context.Context, imageRef reference.DockerImageReference) error { + var icspList []operatorv1alpha1.ImageContentSourcePolicy + var err error + if len(s.icspFile) > 0 { + icspList, err = s.addICSPsFromFile() + if err != nil { + return err + } + } else { + // log errors from accessing cluster since this function has no way of knowing whether it should or should not succeed + icspList, err = s.addICSPsFromCluster(ctx) + if err != nil { + klog.V(4).Infof("No alternative image sources gathered from cluster: %v", err) + } + } + imageRefList, err := s.alternativeImageSources(imageRef, icspList) + if err != nil { + return err + } + s.alternates = imageRefList + return nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 37032f1c52..fa6c71832a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -427,7 +427,7 @@ github.com/opencontainers/image-spec/specs-go/v1 github.com/opencontainers/runc/libcontainer/user # github.com/opencontainers/runtime-spec v1.0.3-0.20200520003142-237cc4f519e2 github.com/opencontainers/runtime-spec/specs-go -# github.com/openshift/api v0.0.0-20210105115604-44119421ec6b => github.com/soltysh/api v0.0.0-20210329125043-97dfec49c179 +# github.com/openshift/api v0.0.0-20210331193751-3acddb19d360 => github.com/soltysh/api v0.0.0-20210329125043-97dfec49c179 ## explicit github.com/openshift/api github.com/openshift/api/annotations @@ -493,7 +493,7 @@ github.com/openshift/build-machinery-go/make/targets/golang github.com/openshift/build-machinery-go/make/targets/openshift github.com/openshift/build-machinery-go/make/targets/openshift/operator github.com/openshift/build-machinery-go/scripts -# github.com/openshift/client-go v0.0.0-20201214125552-e615e336eb49 => github.com/soltysh/client-go v0.0.0-20210329131047-e53d4b5c79d3 +# github.com/openshift/client-go v0.0.0-20210331195552-cf6c2669e01f => github.com/soltysh/client-go v0.0.0-20210329131047-e53d4b5c79d3 ## explicit github.com/openshift/client-go/apps/clientset/versioned github.com/openshift/client-go/apps/clientset/versioned/fake @@ -556,7 +556,7 @@ github.com/openshift/client-go/user/clientset/versioned/fake github.com/openshift/client-go/user/clientset/versioned/scheme github.com/openshift/client-go/user/clientset/versioned/typed/user/v1 github.com/openshift/client-go/user/clientset/versioned/typed/user/v1/fake -# github.com/openshift/library-go v0.0.0-20210219155623-0260bfd7946b => github.com/soltysh/library-go v0.0.0-20210329143525-072267446dd3 +# github.com/openshift/library-go v0.0.0-20210219155623-0260bfd7946b => /home/somalley/code/gowork/src/github.com/openshift/library-go ## explicit github.com/openshift/library-go/pkg/apps/appsserialization github.com/openshift/library-go/pkg/apps/appsutil @@ -578,6 +578,7 @@ github.com/openshift/library-go/pkg/image/internal/reference github.com/openshift/library-go/pkg/image/reference github.com/openshift/library-go/pkg/image/referencemutator github.com/openshift/library-go/pkg/image/registryclient +github.com/openshift/library-go/pkg/image/strategy github.com/openshift/library-go/pkg/image/trigger github.com/openshift/library-go/pkg/legacyapi/legacygroupification github.com/openshift/library-go/pkg/manifest @@ -1384,7 +1385,7 @@ sigs.k8s.io/yaml # github.com/apcera/gssapi => github.com/openshift/gssapi v0.0.0-20161010215902-5fb4217df13b # github.com/openshift/api => github.com/soltysh/api v0.0.0-20210329125043-97dfec49c179 # github.com/openshift/client-go => github.com/soltysh/client-go v0.0.0-20210329131047-e53d4b5c79d3 -# github.com/openshift/library-go => github.com/soltysh/library-go v0.0.0-20210329143525-072267446dd3 +# github.com/openshift/library-go => /home/somalley/code/gowork/src/github.com/openshift/library-go # k8s.io/apimachinery => github.com/openshift/kubernetes-apimachinery v0.0.0-20210318140035-c39220d4515a # k8s.io/cli-runtime => github.com/openshift/kubernetes-cli-runtime v0.0.0-20210323194726-bd1440067d42 # k8s.io/client-go => github.com/openshift/kubernetes-client-go v0.0.0-20210318140334-0e99c560fb6e