Merge branch 'main' into org-oauth2

Author: wxiaoguang

.drone.yml

@@ -571,7 +571,7 @@ steps:
 
   # TODO: We should probably build all dependencies into a test image
   - name: test-e2e
-    image: mcr.microsoft.com/playwright:v1.24.0-focal
+    image: mcr.microsoft.com/playwright:v1.26.1-focal
     commands:
       - curl -sLO https://go.dev/dl/go1.19.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.19.linux-amd64.tar.gz
       - groupadd --gid 1001 gitea && useradd -m --gid 1001 --uid 1001 gitea

docs/content/doc/developers/guidelines-backend.md

@@ -67,22 +67,18 @@ Some actions should allow for rollback when database record insertion/update/del
 So services must be allowed to create a database transaction. Here is some example,
 
 ```go
-// servcies/repository/repo.go
-func CreateXXXX() error {\
-  ctx, committer, err := db.TxContext()
-	if err != nil {
-		return err
-	}
-	defer committer.Close()
-
-  // do something, if return err, it will rollback automatically when `committer.Close()` is invoked.
-  if err := issues.UpdateIssue(ctx, repoID); err != nil {
-      // ...
-  }
-
-  // ......
-
-  return committer.Commit()
+// services/repository/repository.go
+func CreateXXXX() error {
+    return db.WithTx(func(ctx context.Context) error {
+        e := db.GetEngine(ctx)
+        // do something, if err is returned, it will rollback automatically
+        if err := issues.UpdateIssue(ctx, repoID); err != nil {
+            // ...
+            return err
+        }
+        // ...
+        return nil
+    })
 }
 ```
 
@@ -94,14 +90,14 @@ If the function will be used in the transaction, just let `context.Context` as t
 func UpdateIssue(ctx context.Context, repoID int64) error {
     e := db.GetEngine(ctx)
 
-    // ......
+    // ...
 }
 ```
 
 ### Package Name
 
 For the top level package, use a plural as package name, i.e. `services`, `models`, for sub packages, use singular,
-i.e. `servcies/user`, `models/repository`.
+i.e. `services/user`, `models/repository`.
 
 ### Import Alias
 

jest.config.js

@@ -1,3 +1,4 @@
+// to run tests with ES6 module, node must run with "--experimental-vm-modules", or see Makefile's "test-frontend" for reference
 export default {
   rootDir: 'web_src',
   setupFilesAfterEnv: ['jest-extended/all'],
@@ -7,6 +8,8 @@ export default {
   transform: {
     '\\.svg$': '<rootDir>/js/testUtils/jestRawLoader.js',
   },
+  setupFiles: [
+    './js/testUtils/jestSetup.js', // prepare global variables used by our code (eg: window.config)
+  ],
   verbose: false,
 };
-

models/migrations/migrations.go

@@ -413,6 +413,8 @@ var migrations = []Migration{
 	NewMigration("Add badges to users", createUserBadgesTable),
 	// v225 -> v226
 	NewMigration("Alter gpg_key/public_key content TEXT fields to MEDIUMTEXT", alterPublicGPGKeyContentFieldsToMediumText),
+	// v226 -> v227
+	NewMigration("Conan and generic packages do not need to be semantically versioned", fixPackageSemverField),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v226.go

@@ -0,0 +1,15 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"xorm.io/builder"
+	"xorm.io/xorm"
+)
+
+func fixPackageSemverField(x *xorm.Engine) error {
+	_, err := x.Exec(builder.Update(builder.Eq{"semver_compatible": false}).From("`package`").Where(builder.In("`type`", "conan", "generic")))
+	return err
+}

modules/convert/repository.go

@@ -56,9 +56,10 @@ func innerToRepo(repo *repo_model.Repository, mode perm.AccessMode, isParent boo
 		config := unit.ExternalTrackerConfig()
 		hasIssues = true
 		externalTracker = &api.ExternalTracker{
-			ExternalTrackerURL:    config.ExternalTrackerURL,
-			ExternalTrackerFormat: config.ExternalTrackerFormat,
-			ExternalTrackerStyle:  config.ExternalTrackerStyle,
+			ExternalTrackerURL:           config.ExternalTrackerURL,
+			ExternalTrackerFormat:        config.ExternalTrackerFormat,
+			ExternalTrackerStyle:         config.ExternalTrackerStyle,
+			ExternalTrackerRegexpPattern: config.ExternalTrackerRegexpPattern,
 		}
 	}
 	hasWiki := false

modules/git/parse_nogogit.go

@@ -22,70 +22,72 @@ func ParseTreeEntries(data []byte) ([]*TreeEntry, error) {
 	return parseTreeEntries(data, nil)
 }
 
+var sepSpace = []byte{' '}
+
 func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
-	entries := make([]*TreeEntry, 0, 10)
+	var err error
+	entries := make([]*TreeEntry, 0, bytes.Count(data, []byte{'\n'})+1)
 	for pos := 0; pos < len(data); {
-		// expect line to be of the form "<mode> <type> <sha> <space-padded-size>\t<filename>"
+		// expect line to be of the form:
+		// <mode> <type> <sha> <space-padded-size>\t<filename>
+		// <mode> <type> <sha>\t<filename>
+		posEnd := bytes.IndexByte(data[pos:], '\n')
+		if posEnd == -1 {
+			posEnd = len(data)
+		} else {
+			posEnd += pos
+		}
+		line := data[pos:posEnd]
+		posTab := bytes.IndexByte(line, '\t')
+		if posTab == -1 {
+			return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line)
+		}
+
 		entry := new(TreeEntry)
 		entry.ptree = ptree
-		if pos+6 > len(data) {
-			return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data))
+
+		entryAttrs := line[:posTab]
+		entryName := line[posTab+1:]
+
+		entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
+		_ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type
+		entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
+		if len(entryAttrs) > 0 {
+			entrySize := entryAttrs // the last field is the space-padded-size
+			entry.size, _ = strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64)
+			entry.sized = true
 		}
-		switch string(data[pos : pos+6]) {
+
+		switch string(entryMode) {
 		case "100644":
 			entry.entryMode = EntryModeBlob
-			pos += 12 // skip over "100644 blob "
 		case "100755":
 			entry.entryMode = EntryModeExec
-			pos += 12 // skip over "100755 blob "
 		case "120000":
 			entry.entryMode = EntryModeSymlink
-			pos += 12 // skip over "120000 blob "
 		case "160000":
 			entry.entryMode = EntryModeCommit
-			pos += 14 // skip over "160000 object "
 		case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons
 			entry.entryMode = EntryModeTree
-			pos += 12 // skip over "040000 tree "
 		default:
-			return nil, fmt.Errorf("unknown type: %v", string(data[pos:pos+6]))
+			return nil, fmt.Errorf("unknown type: %v", string(entryMode))
 		}
 
-		if pos+40 > len(data) {
-			return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data))
-		}
-		id, err := NewIDFromString(string(data[pos : pos+40]))
+		entry.ID, err = NewIDFromString(string(entryObjectID))
 		if err != nil {
-			return nil, fmt.Errorf("Invalid ls-tree output: %v", err)
-		}
-		entry.ID = id
-		pos += 41 // skip over sha and trailing space
-
-		end := pos + bytes.IndexByte(data[pos:], '\t')
-		if end < pos {
-			return nil, fmt.Errorf("Invalid ls-tree -l output: %s", string(data))
-		}
-		entry.size, _ = strconv.ParseInt(strings.TrimSpace(string(data[pos:end])), 10, 64)
-		entry.sized = true
-
-		pos = end + 1
-
-		end = pos + bytes.IndexByte(data[pos:], '\n')
-		if end < pos {
-			return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data))
+			return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err)
 		}
 
-		// In case entry name is surrounded by double quotes(it happens only in git-shell).
-		if data[pos] == '"' {
-			entry.name, err = strconv.Unquote(string(data[pos:end]))
+		if len(entryName) > 0 && entryName[0] == '"' {
+			entry.name, err = strconv.Unquote(string(entryName))
 			if err != nil {
-				return nil, fmt.Errorf("Invalid ls-tree output: %v", err)
+				return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err)
 			}
 		} else {
-			entry.name = string(data[pos:end])
+			entry.name = string(entryName)
 		}
 
-		pos = end + 1
+		pos = posEnd + 1
 		entries = append(entries, entry)
 	}
 	return entries, nil

modules/git/parse_nogogit_test.go

@@ -12,7 +12,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestParseTreeEntries(t *testing.T) {
+func TestParseTreeEntriesLong(t *testing.T) {
 	testCases := []struct {
 		Input    string
 		Expected []*TreeEntry
@@ -59,11 +59,47 @@ func TestParseTreeEntries(t *testing.T) {
 		assert.NoError(t, err)
 		assert.Len(t, entries, len(testCase.Expected))
 		for i, entry := range entries {
-			assert.EqualValues(t, testCase.Expected[i].ID, entry.ID)
-			assert.EqualValues(t, testCase.Expected[i].name, entry.name)
-			assert.EqualValues(t, testCase.Expected[i].entryMode, entry.entryMode)
-			assert.EqualValues(t, testCase.Expected[i].sized, entry.sized)
-			assert.EqualValues(t, testCase.Expected[i].size, entry.size)
+			assert.EqualValues(t, testCase.Expected[i], entry)
 		}
 	}
 }
+
+func TestParseTreeEntriesShort(t *testing.T) {
+	testCases := []struct {
+		Input    string
+		Expected []*TreeEntry
+	}{
+		{
+			Input: `100644 blob ea0d83c9081af9500ac9f804101b3fd0a5c293af	README.md
+040000 tree 84b90550547016f73c5dd3f50dea662389e67b6d	assets
+`,
+			Expected: []*TreeEntry{
+				{
+					ID:        MustIDFromString("ea0d83c9081af9500ac9f804101b3fd0a5c293af"),
+					name:      "README.md",
+					entryMode: EntryModeBlob,
+				},
+				{
+					ID:        MustIDFromString("84b90550547016f73c5dd3f50dea662389e67b6d"),
+					name:      "assets",
+					entryMode: EntryModeTree,
+				},
+			},
+		},
+	}
+	for _, testCase := range testCases {
+		entries, err := ParseTreeEntries([]byte(testCase.Input))
+		assert.NoError(t, err)
+		assert.Len(t, entries, len(testCase.Expected))
+		for i, entry := range entries {
+			assert.EqualValues(t, testCase.Expected[i], entry)
+		}
+	}
+}
+
+func TestParseTreeEntriesInvalid(t *testing.T) {
+	// there was a panic: "runtime error: slice bounds out of range" when the input was invalid: #20315
+	entries, err := ParseTreeEntries([]byte("100644 blob ea0d83c9081af9500ac9f804101b3fd0a5c293af"))
+	assert.Error(t, err)
+	assert.Len(t, entries, 0)
+}

modules/git/repo_language_stats_nogogit.go

@@ -57,7 +57,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 
 	tree := commit.Tree
 
-	entries, err := tree.ListEntriesRecursive()
+	entries, err := tree.ListEntriesRecursiveWithSize()
 	if err != nil {
 		return nil, err
 	}

modules/git/tree_gogit.go

@@ -57,8 +57,8 @@ func (t *Tree) ListEntries() (Entries, error) {
 	return entries, nil
 }
 
-// ListEntriesRecursive returns all entries of current tree recursively including all subtrees
-func (t *Tree) ListEntriesRecursive() (Entries, error) {
+// ListEntriesRecursiveWithSize returns all entries of current tree recursively including all subtrees
+func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) {
 	if t.gogitTree == nil {
 		err := t.loadTreeObject()
 		if err != nil {
@@ -92,3 +92,8 @@ func (t *Tree) ListEntriesRecursive() (Entries, error) {
 
 	return entries, nil
 }
+
+// ListEntriesRecursiveFast is the alias of ListEntriesRecursiveWithSize for the gogit version
+func (t *Tree) ListEntriesRecursiveFast() (Entries, error) {
+	return t.ListEntriesRecursiveWithSize()
+}

modules/git/tree_nogogit.go

@@ -99,13 +99,16 @@ func (t *Tree) ListEntries() (Entries, error) {
 	return t.entries, err
 }
 
-// ListEntriesRecursive returns all entries of current tree recursively including all subtrees
-func (t *Tree) ListEntriesRecursive() (Entries, error) {
+// listEntriesRecursive returns all entries of current tree recursively including all subtrees
+// extraArgs could be "-l" to get the size, which is slower
+func (t *Tree) listEntriesRecursive(extraArgs ...string) (Entries, error) {
 	if t.entriesRecursiveParsed {
 		return t.entriesRecursive, nil
 	}
 
-	stdout, _, runErr := NewCommand(t.repo.Ctx, "ls-tree", "-t", "-l", "-r", t.ID.String()).RunStdBytes(&RunOpts{Dir: t.repo.Path})
+	args := append([]string{"ls-tree", "-t", "-r"}, extraArgs...)
+	args = append(args, t.ID.String())
+	stdout, _, runErr := NewCommand(t.repo.Ctx, args...).RunStdBytes(&RunOpts{Dir: t.repo.Path})
 	if runErr != nil {
 		return nil, runErr
 	}
@@ -118,3 +121,13 @@ func (t *Tree) ListEntriesRecursive() (Entries, error) {
 
 	return t.entriesRecursive, err
 }
+
+// ListEntriesRecursiveFast returns all entries of current tree recursively including all subtrees, no size
+func (t *Tree) ListEntriesRecursiveFast() (Entries, error) {
+	return t.listEntriesRecursive()
+}
+
+// ListEntriesRecursiveWithSize returns all entries of current tree recursively including all subtrees, with size
+func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) {
+	return t.listEntriesRecursive("--long")
+}

modules/packages/npm/creator.go

@@ -66,7 +66,8 @@ type PackageMetadata struct {
 	License        string                             `json:"license,omitempty"`
 }
 
-// PackageMetadataVersion https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version
+// PackageMetadataVersion documentation: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version
+// PackageMetadataVersion response: https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object
 type PackageMetadataVersion struct {
 	ID                   string              `json:"_id"`
 	Name                 string              `json:"name"`
@@ -80,6 +81,7 @@ type PackageMetadataVersion struct {
 	Dependencies         map[string]string   `json:"dependencies,omitempty"`
 	DevDependencies      map[string]string   `json:"devDependencies,omitempty"`
 	PeerDependencies     map[string]string   `json:"peerDependencies,omitempty"`
+	Bin                  map[string]string   `json:"bin,omitempty"`
 	OptionalDependencies map[string]string   `json:"optionalDependencies,omitempty"`
 	Readme               string              `json:"readme,omitempty"`
 	Dist                 PackageDistribution `json:"dist"`
@@ -220,6 +222,7 @@ func ParsePackage(r io.Reader) (*Package, error) {
 				DevelopmentDependencies: meta.DevDependencies,
 				PeerDependencies:        meta.PeerDependencies,
 				OptionalDependencies:    meta.OptionalDependencies,
+				Bin:                     meta.Bin,
 				Readme:                  meta.Readme,
 			},
 		}