Skip to content

Commit e07ddcb

Browse files
wxiaoguangwxiaoguang
authored
Merge branch 'main' into pacman-packages
2 parents d679da8 + 145e266 commit e07ddcb

File tree

66 files changed

+808
-609
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+808
-609
lines changed

‎models/activities/action.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,10 @@ func (a *Action) TableIndices() []*schemas.Index {
171171
cudIndex := schemas.NewIndex("c_u_d", schemas.IndexType)
172172
cudIndex.AddColumn("created_unix", "user_id", "is_deleted")
173173

174-
indices := []*schemas.Index{actUserIndex, repoIndex, cudIndex}
174+
cuIndex := schemas.NewIndex("c_u", schemas.IndexType)
175+
cuIndex.AddColumn("user_id", "is_deleted")
176+
177+
indices := []*schemas.Index{actUserIndex, repoIndex, cudIndex, cuIndex}
175178

176179
return indices
177180
}

‎models/migrations/migrations.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -365,6 +365,7 @@ func prepareMigrationTasks() []*migration {
365365
newMigration(305, "Add Repository Licenses", v1_23.AddRepositoryLicenses),
366366
newMigration(306, "Add BlockAdminMergeOverride to ProtectedBranch", v1_23.AddBlockAdminMergeOverrideBranchProtection),
367367
newMigration(307, "Fix milestone deadline_unix when there is no due date", v1_23.FixMilestoneNoDueDate),
368+
newMigration(308, "Add index(user_id, is_deleted) for action table", v1_23.AddNewIndexForUserDashboard),
368369
}
369370
return preparedMigrations
370371
}

‎models/migrations/v1_23/v308.go

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
// Copyright 2024 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package v1_23 //nolint
5+
6+
import (
7+
"code.gitea.io/gitea/modules/timeutil"
8+
9+
"xorm.io/xorm"
10+
"xorm.io/xorm/schemas"
11+
)
12+
13+
type improveActionTableIndicesAction struct {
14+
ID int64 `xorm:"pk autoincr"`
15+
UserID int64 `xorm:"INDEX"` // Receiver user id.
16+
OpType int
17+
ActUserID int64 // Action user id.
18+
RepoID int64
19+
CommentID int64 `xorm:"INDEX"`
20+
IsDeleted bool `xorm:"NOT NULL DEFAULT false"`
21+
RefName string
22+
IsPrivate bool `xorm:"NOT NULL DEFAULT false"`
23+
Content string `xorm:"TEXT"`
24+
CreatedUnix timeutil.TimeStamp `xorm:"created"`
25+
}
26+
27+
// TableName sets the name of this table
28+
func (*improveActionTableIndicesAction) TableName() string {
29+
return "action"
30+
}
31+
32+
func (a *improveActionTableIndicesAction) TableIndices() []*schemas.Index {
33+
repoIndex := schemas.NewIndex("r_u_d", schemas.IndexType)
34+
repoIndex.AddColumn("repo_id", "user_id", "is_deleted")
35+
36+
actUserIndex := schemas.NewIndex("au_r_c_u_d", schemas.IndexType)
37+
actUserIndex.AddColumn("act_user_id", "repo_id", "created_unix", "user_id", "is_deleted")
38+
39+
cudIndex := schemas.NewIndex("c_u_d", schemas.IndexType)
40+
cudIndex.AddColumn("created_unix", "user_id", "is_deleted")
41+
42+
cuIndex := schemas.NewIndex("c_u", schemas.IndexType)
43+
cuIndex.AddColumn("user_id", "is_deleted")
44+
45+
indices := []*schemas.Index{actUserIndex, repoIndex, cudIndex, cuIndex}
46+
47+
return indices
48+
}
49+
50+
func AddNewIndexForUserDashboard(x *xorm.Engine) error {
51+
return x.Sync(new(improveActionTableIndicesAction))
52+
}

‎models/perm/access_mode.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,3 +60,6 @@ func ParseAccessMode(permission string, allowed ...AccessMode) AccessMode {
6060
}
6161
return util.Iif(slices.Contains(allowed, m), m, AccessModeNone)
6262
}
63+
64+
// ErrInvalidAccessMode is returned when an invalid access mode is used
65+
var ErrInvalidAccessMode = util.NewInvalidArgumentErrorf("Invalid access mode")

‎modules/indexer/code/bleve/bleve.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"github.com/blevesearch/bleve/v2/analysis/token/camelcase"
3232
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
3333
"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
34+
"github.com/blevesearch/bleve/v2/analysis/tokenizer/letter"
3435
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
3536
"github.com/blevesearch/bleve/v2/mapping"
3637
"github.com/blevesearch/bleve/v2/search/query"
@@ -69,7 +70,7 @@ const (
6970
filenameIndexerAnalyzer = "filenameIndexerAnalyzer"
7071
filenameIndexerTokenizer = "filenameIndexerTokenizer"
7172
repoIndexerDocType = "repoIndexerDocType"
72-
repoIndexerLatestVersion = 7
73+
repoIndexerLatestVersion = 8
7374
)
7475

7576
// generateBleveIndexMapping generates a bleve index mapping for the repo indexer
@@ -105,7 +106,7 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) {
105106
} else if err := mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]any{
106107
"type": analyzer_custom.Name,
107108
"char_filters": []string{},
108-
"tokenizer": unicode.Name,
109+
"tokenizer": letter.Name,
109110
"token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name},
110111
}); err != nil {
111112
return nil, err

‎modules/indexer/code/elasticsearch/elasticsearch.go

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import (
3030
)
3131

3232
const (
33-
esRepoIndexerLatestVersion = 2
33+
esRepoIndexerLatestVersion = 3
3434
// multi-match-types, currently only 2 types are used
3535
// Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types
3636
esMultiMatchTypeBestFields = "best_fields"
@@ -60,6 +60,10 @@ const (
6060
"settings": {
6161
"analysis": {
6262
"analyzer": {
63+
"content_analyzer": {
64+
"tokenizer": "content_tokenizer",
65+
"filter" : ["lowercase"]
66+
},
6367
"filename_path_analyzer": {
6468
"tokenizer": "path_tokenizer"
6569
},
@@ -68,6 +72,10 @@ const (
6872
}
6973
},
7074
"tokenizer": {
75+
"content_tokenizer": {
76+
"type": "simple_pattern_split",
77+
"pattern": "[^a-zA-Z0-9]"
78+
},
7179
"path_tokenizer": {
7280
"type": "path_hierarchy",
7381
"delimiter": "/"
@@ -104,7 +112,8 @@ const (
104112
"content": {
105113
"type": "text",
106114
"term_vector": "with_positions_offsets",
107-
"index": true
115+
"index": true,
116+
"analyzer": "content_analyzer"
108117
},
109118
"commit_id": {
110119
"type": "keyword",

‎modules/indexer/code/indexer_test.go

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,55 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
181181
},
182182
},
183183
},
184+
// Search for matches on the contents of files regardless of case.
185+
{
186+
RepoIDs: nil,
187+
Keyword: "dESCRIPTION",
188+
Langs: 1,
189+
Results: []codeSearchResult{
190+
{
191+
Filename: "README.md",
192+
Content: "# repo1\n\nDescription for repo1",
193+
},
194+
},
195+
},
196+
// Search for an exact match on the filename within the repo '62' (case insenstive).
197+
// This scenario yields a single result (the file avocado.md on the repo '62')
198+
{
199+
RepoIDs: []int64{62},
200+
Keyword: "AVOCADO.MD",
201+
Langs: 1,
202+
Results: []codeSearchResult{
203+
{
204+
Filename: "avocado.md",
205+
Content: "# repo1\n\npineaple pie of cucumber juice",
206+
},
207+
},
208+
},
209+
// Search for matches on the contents of files when the criteria is a expression.
210+
{
211+
RepoIDs: []int64{62},
212+
Keyword: "console.log",
213+
Langs: 1,
214+
Results: []codeSearchResult{
215+
{
216+
Filename: "example-file.js",
217+
Content: "console.log(\"Hello, World!\")",
218+
},
219+
},
220+
},
221+
// Search for matches on the contents of files when the criteria is part of a expression.
222+
{
223+
RepoIDs: []int64{62},
224+
Keyword: "log",
225+
Langs: 1,
226+
Results: []codeSearchResult{
227+
{
228+
Filename: "example-file.js",
229+
Content: "console.log(\"Hello, World!\")",
230+
},
231+
},
232+
},
184233
}
185234

186235
for _, kw := range keywords {

‎modules/indexer/internal/bleve/util.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,13 @@ package bleve
66
import (
77
"errors"
88
"os"
9+
"unicode"
910

1011
"code.gitea.io/gitea/modules/log"
1112
"code.gitea.io/gitea/modules/util"
1213

1314
"github.com/blevesearch/bleve/v2"
14-
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
15+
unicode_tokenizer "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
1516
"github.com/blevesearch/bleve/v2/index/upsidedown"
1617
"github.com/ethantkoenig/rupture"
1718
)
@@ -57,7 +58,7 @@ func openIndexer(path string, latestVersion int) (bleve.Index, int, error) {
5758
// may be different on two string and they still be considered equivalent.
5859
// Given a phrasse, its shortest word determines its fuzziness. If a phrase uses CJK (eg: `갃갃갃` `啊啊啊`), the fuzziness is zero.
5960
func GuessFuzzinessByKeyword(s string) int {
60-
tokenizer := unicode.NewUnicodeTokenizer()
61+
tokenizer := unicode_tokenizer.NewUnicodeTokenizer()
6162
tokens := tokenizer.Tokenize([]byte(s))
6263

6364
if len(tokens) > 0 {
@@ -77,8 +78,10 @@ func guessFuzzinessByKeyword(s string) int {
7778
// according to https://github.com/blevesearch/bleve/issues/1563, the supported max fuzziness is 2
7879
// magic number 4 was chosen to determine the levenshtein distance per each character of a keyword
7980
// BUT, when using CJK (eg: `갃갃갃` `啊啊啊`), it mismatches a lot.
81+
// Likewise, queries whose terms contains characters that are *not* letters should not use fuzziness
82+
8083
for _, r := range s {
81-
if r >= 128 {
84+
if r >= 128 || !unicode.IsLetter(r) {
8285
return 0
8386
}
8487
}

‎modules/indexer/internal/bleve/util_test.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,14 @@ func TestBleveGuessFuzzinessByKeyword(t *testing.T) {
3535
Input: "갃갃갃",
3636
Fuzziness: 0,
3737
},
38+
{
39+
Input: "repo1",
40+
Fuzziness: 0,
41+
},
42+
{
43+
Input: "avocado.md",
44+
Fuzziness: 0,
45+
},
3846
}
3947

4048
for _, scenario := range scenarios {

‎modules/markup/html.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,10 @@ func createLink(href, content, class string) *html.Node {
442442
a := &html.Node{
443443
Type: html.ElementNode,
444444
Data: atom.A.String(),
445-
Attr: []html.Attribute{{Key: "href", Val: href}},
445+
Attr: []html.Attribute{
446+
{Key: "href", Val: href},
447+
{Key: "data-markdown-generated-content"},
448+
},
446449
}
447450

448451
if class != "" {

‎modules/markup/html_codepreview_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,5 @@ func TestRenderCodePreview(t *testing.T) {
3030
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
3131
}
3232
test("http://localhost:3000/owner/repo/src/commit/0123456789/foo/bar.md#L10-L20", "<p><div>code preview</div></p>")
33-
test("http://other/owner/repo/src/commit/0123456789/foo/bar.md#L10-L20", `<p><a href="http://other/owner/repo/src/commit/0123456789/foo/bar.md#L10-L20" rel="nofollow">http://other/owner/repo/src/commit/0123456789/foo/bar.md#L10-L20</a></p>`)
33+
test("http://other/owner/repo/src/commit/0123456789/foo/bar.md#L10-L20", `<p><a href="http://other/owner/repo/src/commit/0123456789/foo/bar.md#L10-L20" data-markdown-generated-content="" rel="nofollow">http://other/owner/repo/src/commit/0123456789/foo/bar.md#L10-L20</a></p>`)
3434
}

‎modules/markup/html_internal_test.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,9 @@ func numericIssueLink(baseURL, class string, index int, marker string) string {
3333

3434
// link an HTML link
3535
func link(href, class, contents string) string {
36-
if class != "" {
37-
class = " class=\"" + class + "\""
38-
}
39-
40-
return fmt.Sprintf("<a href=\"%s\"%s>%s</a>", href, class, contents)
36+
extra := ` data-markdown-generated-content=""`
37+
extra += util.Iif(class != "", ` class="`+class+`"`, "")
38+
return fmt.Sprintf(`<a href="%s"%s>%s</a>`, href, extra, contents)
4139
}
4240

4341
var numericMetas = map[string]string{
@@ -353,7 +351,9 @@ func TestRender_FullIssueURLs(t *testing.T) {
353351
Metas: localMetas,
354352
}, []processor{fullIssuePatternProcessor}, strings.NewReader(input), &result)
355353
assert.NoError(t, err)
356-
assert.Equal(t, expected, result.String())
354+
actual := result.String()
355+
actual = strings.ReplaceAll(actual, ` data-markdown-generated-content=""`, "")
356+
assert.Equal(t, expected, actual)
357357
}
358358
test("Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6",
359359
"Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6")

‎modules/markup/html_test.go

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,9 @@ func TestRender_CrossReferences(t *testing.T) {
116116
Metas: localMetas,
117117
}, input)
118118
assert.NoError(t, err)
119-
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
119+
actual := strings.TrimSpace(buffer)
120+
actual = strings.ReplaceAll(actual, ` data-markdown-generated-content=""`, "")
121+
assert.Equal(t, strings.TrimSpace(expected), actual)
120122
}
121123

122124
test(
@@ -156,7 +158,9 @@ func TestRender_links(t *testing.T) {
156158
},
157159
}, input)
158160
assert.NoError(t, err)
159-
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
161+
actual := strings.TrimSpace(buffer)
162+
actual = strings.ReplaceAll(actual, ` data-markdown-generated-content=""`, "")
163+
assert.Equal(t, strings.TrimSpace(expected), actual)
160164
}
161165

162166
oldCustomURLSchemes := setting.Markdown.CustomURLSchemes
@@ -267,7 +271,9 @@ func TestRender_email(t *testing.T) {
267271
},
268272
}, input)
269273
assert.NoError(t, err)
270-
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res))
274+
actual := strings.TrimSpace(res)
275+
actual = strings.ReplaceAll(actual, ` data-markdown-generated-content=""`, "")
276+
assert.Equal(t, strings.TrimSpace(expected), actual)
271277
}
272278
// Text that should be turned into email link
273279

@@ -616,7 +622,9 @@ func TestPostProcess_RenderDocument(t *testing.T) {
616622
Metas: localMetas,
617623
}, strings.NewReader(input), &res)
618624
assert.NoError(t, err)
619-
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res.String()))
625+
actual := strings.TrimSpace(res.String())
626+
actual = strings.ReplaceAll(actual, ` data-markdown-generated-content=""`, "")
627+
assert.Equal(t, strings.TrimSpace(expected), actual)
620628
}
621629

622630
// Issue index shouldn't be post processing in a document.

‎modules/markup/markdown/markdown_test.go

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -311,7 +311,8 @@ func TestTotal_RenderWiki(t *testing.T) {
311311
IsWiki: true,
312312
}, sameCases[i])
313313
assert.NoError(t, err)
314-
assert.Equal(t, template.HTML(answers[i]), line)
314+
actual := strings.ReplaceAll(string(line), ` data-markdown-generated-content=""`, "")
315+
assert.Equal(t, answers[i], actual)
315316
}
316317

317318
testCases := []string{
@@ -336,7 +337,8 @@ func TestTotal_RenderWiki(t *testing.T) {
336337
IsWiki: true,
337338
}, testCases[i])
338339
assert.NoError(t, err)
339-
assert.Equal(t, template.HTML(testCases[i+1]), line)
340+
actual := strings.ReplaceAll(string(line), ` data-markdown-generated-content=""`, "")
341+
assert.EqualValues(t, testCases[i+1], actual)
340342
}
341343
}
342344

@@ -356,7 +358,8 @@ func TestTotal_RenderString(t *testing.T) {
356358
Metas: localMetas,
357359
}, sameCases[i])
358360
assert.NoError(t, err)
359-
assert.Equal(t, template.HTML(answers[i]), line)
361+
actual := strings.ReplaceAll(string(line), ` data-markdown-generated-content=""`, "")
362+
assert.Equal(t, answers[i], actual)
360363
}
361364

362365
testCases := []string{}
@@ -996,7 +999,8 @@ space</p>
996999
for i, c := range cases {
9971000
result, err := markdown.RenderString(&markup.RenderContext{Ctx: context.Background(), Links: c.Links, IsWiki: c.IsWiki}, input)
9981001
assert.NoError(t, err, "Unexpected error in testcase: %v", i)
999-
assert.Equal(t, c.Expected, string(result), "Unexpected result in testcase %v", i)
1002+
actual := strings.ReplaceAll(string(result), ` data-markdown-generated-content=""`, "")
1003+
assert.Equal(t, c.Expected, actual, "Unexpected result in testcase %v", i)
10001004
}
10011005
}
10021006

‎modules/markup/sanitizer_default.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ func (st *Sanitizer) createDefaultPolicy() *bluemonday.Policy {
107107
"start", "summary", "tabindex", "target",
108108
"title", "type", "usemap", "valign", "value",
109109
"vspace", "width", "itemprop",
110+
"data-markdown-generated-content",
110111
}
111112

112113
generalSafeElements := []string{

0 commit comments

Comments
 (0)