feat: unify pull cli

Signed-off-by: zongz <[email protected]>

Author: zong-zhe

cmd/kcl/commands/args.go

@@ -1,8 +1,104 @@
 package cmd
 
+import (
+	"net/url"
+
+	"kcl-lang.io/kpm/pkg/client"
+	"kcl-lang.io/kpm/pkg/constants"
+	"kcl-lang.io/kpm/pkg/opt"
+)
+
 func argsGet(a []string, n int) string {
 	if len(a) > n {
 		return a[n]
 	}
 	return ""
 }
+
+func ParseUrlFromArgs(cli *client.KpmClient, args []string) (*url.URL, error) {
+	var sourceUrl url.URL
+
+	if len(args) == 0 {
+		if len(git) != 0 {
+			gitUrl, err := url.Parse(git)
+			if err != nil {
+				return nil, err
+			}
+
+			gitUrl.Scheme = constants.GitScheme
+			query := gitUrl.Query()
+			if tag != "" {
+				query.Add(constants.Tag, tag)
+			}
+			if commit != "" {
+				query.Add(constants.GitCommit, commit)
+			}
+			if branch != "" {
+				query.Add(constants.GitBranch, branch)
+			}
+			gitUrl.RawQuery = query.Encode()
+			sourceUrl = *gitUrl
+		} else if len(oci) != 0 {
+			ociUrl, err := url.Parse(oci)
+			if err != nil {
+				return nil, err
+			}
+
+			ociUrl.Scheme = constants.OciScheme
+			query := ociUrl.Query()
+			query.Add(constants.Tag, tag)
+			ociUrl.RawQuery = query.Encode()
+			sourceUrl = *ociUrl
+		}
+	} else {
+		url, err := url.Parse(args[0])
+		if err != nil {
+			return nil, err
+		}
+		query := url.Query()
+		url.Opaque = ""
+		regOpts, err := opt.NewRegistryOptionsFrom(args[0], cli.GetSettings())
+		if err != nil {
+			return nil, err
+		}
+
+		if regOpts.Git != nil {
+			if url.Scheme != constants.GitScheme && url.Scheme != constants.SshScheme {
+				url.Scheme = constants.GitScheme
+			}
+			if tag != "" {
+				query.Add(constants.Tag, tag)
+			}
+			if commit != "" {
+				query.Add(constants.GitCommit, commit)
+			}
+			if branch != "" {
+				query.Add(constants.GitBranch, branch)
+			}
+		} else if regOpts.Oci != nil {
+			url.Scheme = constants.OciScheme
+			url.Host = regOpts.Oci.Reg
+			url.Path = regOpts.Oci.Repo
+			if regOpts.Oci.Tag != "" {
+				query.Add(constants.Tag, regOpts.Oci.Tag)
+			}
+			if tag != "" {
+				query.Add(constants.Tag, tag)
+			}
+		} else if regOpts.Registry != nil {
+			url.Scheme = constants.DefaultOciScheme
+			url.Host = regOpts.Registry.Reg
+			url.Path = regOpts.Registry.Repo
+			if regOpts.Registry.Tag != "" {
+				query.Add(constants.Tag, regOpts.Registry.Tag)
+			}
+			if tag != "" {
+				query.Add(constants.Tag, tag)
+			}
+		}
+
+		url.RawQuery = query.Encode()
+		sourceUrl = *url
+	}
+	return &sourceUrl, nil
+}

cmd/kcl/commands/mod_pull.go

@@ -3,13 +3,29 @@ package cmd
 import (
 	"github.com/spf13/cobra"
 	"kcl-lang.io/kpm/pkg/client"
+	"kcl-lang.io/kpm/pkg/downloader"
 )
 
 const (
 	modPullDesc = `This command pulls kcl modules from the registry.
 `
 	modPullExample = `  # Pull the the module named "k8s" to the local path from the registry
-  kcl mod pull k8s`
+  kcl mod pull k8s
+
+  # Pull the module dependency named "k8s" with the version "1.28"
+  kcl mod add k8s:1.28
+
+  # Pull the module from the GitHub by git url
+  kcl mod pull git://github.com/kcl-lang/konfig --tag v0.4.0
+
+  # Pull the module from the OCI Registry by oci url
+  kcl mod pull oci://ghcr.io/kcl-lang/helloworld --tag v0.1.0
+
+  # Pull the module from the GitHub by flag
+  kcl mod pull --git https://github.com/kcl-lang/konfig --tag v0.4.0
+
+  # Pull the module from the OCI Registry by flag
+  kcl mod pull --oci https://ghcr.io/kcl-lang/helloworld --tag 0.1.0`
 )
 
 // NewModPullCmd returns the mod pull command.
@@ -20,14 +36,39 @@ func NewModPullCmd(cli *client.KpmClient) *cobra.Command {
 		Long:    modPullDesc,
 		Example: modPullExample,
 		RunE: func(_ *cobra.Command, args []string) error {
-			source := argsGet(args, 0)
 			localPath := argsGet(args, 1)
-			return cli.PullFromOci(localPath, source, tag)
+			return pull(cli, args, localPath)
 		},
 		SilenceUsage: true,
 	}
 
-	cmd.Flags().StringVar(&tag, "tag", "", "git repository tag")
+	cmd.Flags().StringVar(&git, "git", "", "git repository url")
+	cmd.Flags().StringVar(&oci, "oci", "", "oci repository url")
+	cmd.Flags().StringVar(&tag, "tag", "", "git or oci repository tag")
+	cmd.Flags().StringVar(&commit, "commit", "", "git repository commit")
+	cmd.Flags().StringVar(&branch, "branch", "", "git repository branch")
 
 	return cmd
 }
+
+func pull(cli *client.KpmClient, args []string, localPath string) error {
+	sourceUrl, err := ParseUrlFromArgs(cli, args)
+	if err != nil {
+		return err
+	}
+	source, err := downloader.NewSourceFromStr(sourceUrl.String())
+	if err != nil {
+		return err
+	}
+
+	_, err = cli.Pull(
+		client.WithPullSource(source),
+		client.WithLocalPath(localPath),
+	)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/kcl/commands/mod_push.go

@@ -169,7 +169,7 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, cli *client
 			"only support url scheme 'oci://'.",
 		)
 	}
-	ociOpts.Annotations, err = kpmoci.GenOciManifestFromPkg(kclPkg)
+	ociOpts.Annotations, err = kclPkg.GenOciManifestFromPkg()
 	if err != nil {
 		return err
 	}

go.mod

@@ -30,6 +30,7 @@ require (
 	github.com/containers/ocicrypt v1.1.10 // indirect
 	github.com/containers/storage v1.54.0 // indirect
 	github.com/distribution/reference v0.6.0 // indirect
+	github.com/elliotchance/orderedmap/v2 v2.2.0 // indirect
 	github.com/emicklei/proto v1.13.2 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
@@ -208,3 +209,8 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
+
+replace (
+	kcl-lang.io/kpm => /Users/zongz/Workspace/kusionstack/kpm
+	kcl-lang.io/kpm v0.9.0-rc.2 => kcl-lang.io/kpm v0.9.0-rc.2.0.20240704123051-efc7b9c78ae8
+)

go.sum

@@ -373,6 +373,8 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
+github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk=
+github.com/elliotchance/orderedmap/v2 v2.2.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/proto v1.13.2 h1:z/etSFO3uyXeuEsVPzfl56WNgzcvIr42aQazXaQmFZY=
@@ -1689,8 +1691,8 @@ kcl-lang.io/kcl-openapi v0.6.3 h1:6Br0IBaNshRvBfmaH22Zv1miVEt6FWlmh2v/wBLb194=
 kcl-lang.io/kcl-openapi v0.6.3/go.mod h1:Ai9mFztCVKkRSFabczO/r5hCNdqaNtAc2ZIRxTeV0Mk=
 kcl-lang.io/kcl-playground v0.5.1 h1:MKQQUHgt4+2QyU2NVwa73oksOaBJGDi4keGoggA0MiU=
 kcl-lang.io/kcl-playground v0.5.1/go.mod h1:IFmnlw7m011ccX8OidMUfnnN2u/TWdtQGxyABRTbmow=
-kcl-lang.io/kpm v0.9.0-rc.2 h1:0v76+Tk0XnWawXHLoK+XuGTp+7QxD+vNUCzteIgNE0w=
-kcl-lang.io/kpm v0.9.0-rc.2/go.mod h1:mwEU/1fURutTNaO1SeGpoFh1iMf+Cc6SYyDAuIQHGNg=
+kcl-lang.io/kpm v0.9.0-rc.2.0.20240704123051-efc7b9c78ae8 h1:HXGw7kK5b4xiTiinA1R10sEOSYWqPvrCFKmZdTbFAyw=
+kcl-lang.io/kpm v0.9.0-rc.2.0.20240704123051-efc7b9c78ae8/go.mod h1:+8dz5o9l1DEzEYdfRetnAXhrfqKdzTQb3s99DubbHhk=
 kcl-lang.io/lib v0.9.0-rc.2 h1:1FcWnHKvxe5TGtEwTtd3CTg6V8aw4VdEWtYvQEp15Gk=
 kcl-lang.io/lib v0.9.0-rc.2/go.mod h1:tu+tzwGgHLzYZSIxUG/ntipStrxZd6OvutWYPTxS7cs=
 oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo=

test/e2e/test_suites/test_kcl_mod_pull/stdout

@@ -1,4 +1,4 @@
-start to pull 'helloworld'
-the lastest version '0.1.1' will be pulled
-pulling 'test/helloworld:0.1.1' from 'localhost:5001/test/helloworld'
-pulled 'helloworld' in '<workspace>/test_space/localhost:5001/test/helloworld' successfully
+start to pull oci://localhost:5001/test/helloworld
+the lastest version '0.1.1' will be downloaded
+downloading 'test/helloworld:0.1.1' from 'localhost:5001/test/helloworld:0.1.1'
+pulled helloworld 0.1.1 successfully

test/e2e/test_suites/test_kcl_mod_pull_0/stdout

@@ -1,3 +1,3 @@
-start to pull 'helloworld:0.1.1'
-pulling 'test/helloworld:0.1.1' from 'localhost:5001/test/helloworld'
-pulled 'helloworld:0.1.1' in '<workspace>/test_space/localhost:5001/test/helloworld/0.1.1' successfully
+start to pull oci://localhost:5001/test/helloworld?tag=0.1.1
+downloading 'test/helloworld:0.1.1' from 'localhost:5001/test/helloworld:0.1.1'
+pulled helloworld 0.1.1 successfully

test/e2e/test_suites/test_kcl_mod_pull_1/input

@@ -1 +1 @@
-kcl mod pull oci://localhost:5001/test/helloworld:0.1.1
+kcl mod pull oci://localhost:5001/test/helloworld --tag 0.1.1

test/e2e/test_suites/test_kcl_mod_pull_1/stdout

@@ -1,4 +1,3 @@
-start to pull 'oci://localhost:5001/test/helloworld:0.1.1'
-the lastest version '0.1.1' will be pulled
-pulling '/test/helloworld:0.1.1:0.1.1' from 'localhost:5001/test/helloworld:0.1.1'
-pulled 'oci://localhost:5001/test/helloworld:0.1.1' in '<workspace>/test_space/localhost:5001/test/helloworld:0.1.1' successfully
+start to pull oci://localhost:5001/test/helloworld?tag=0.1.1
+downloading 'test/helloworld:0.1.1' from 'localhost:5001/test/helloworld:0.1.1'
+pulled helloworld 0.1.1 successfully

test/e2e/test_suites/test_kcl_mod_pull_2/input

@@ -0,0 +1 @@
+kcl mod pull git://github.com/kcl-lang/flask-demo-kcl-manifests --commit ade147b

test/e2e/test_suites/test_kcl_mod_pull_2/stderr

@@ -0,0 +1,3 @@
+start to pull https://github.com/kcl-lang/flask-demo-kcl-manifests?commit=ade147b
+cloning 'https://github.com/kcl-lang/flask-demo-kcl-manifests' with commit 'ade147b'
+pulled flask_manifests 0.0.1 successfully

test/e2e/test_suites/test_kcl_mod_pull_2/stdout

@@ -0,0 +1 @@
+kcl mod pull --git https://github.com/kcl-lang/flask-demo-kcl-manifests --branch main

test/e2e/test_suites/test_kcl_mod_pull_3/input

@@ -0,0 +1,3 @@
+start to pull https://github.com/kcl-lang/flask-demo-kcl-manifests?branch=main
+cloning 'https://github.com/kcl-lang/flask-demo-kcl-manifests' 
+pulled flask_manifests 0.0.1 successfully

test/e2e/test_suites/test_kcl_mod_pull_3/stderr

@@ -0,0 +1 @@
+kcl mod pull --oci http://localhost:5001/test/helloworld --tag 0.1.1

test/e2e/test_suites/test_kcl_mod_pull_3/stdout

@@ -0,0 +1,3 @@
+start to pull oci://localhost:5001/test/helloworld?tag=0.1.1
+downloading 'test/helloworld:0.1.1' from 'localhost:5001/test/helloworld:0.1.1'
+pulled helloworld 0.1.1 successfully