From 8650571666f74884fe2ed12b4c6fe2c77ef668ff Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Sat, 4 Feb 2023 14:30:25 +0800 Subject: [PATCH 01/72] chore(actions): support cron schedule task. Signed-off-by: Bo-Yi.Wu --- go.mod | 3 + go.sum | 2 + models/actions/schedule.go | 67 +++++++++ models/actions/schedule_list.go | 95 +++++++++++++ models/migrations/migrations.go | 3 + models/migrations/v1_20/v245.go | 34 +++++ modules/actions/workflows.go | 23 ++- services/actions/init.go | 4 + services/actions/notifier_helper.go | 212 +++++++++++++++++++++++----- services/actions/schedule.go | 63 +++++++++ 10 files changed, 464 insertions(+), 42 deletions(-) create mode 100644 models/actions/schedule.go create mode 100644 models/actions/schedule_list.go create mode 100644 models/migrations/v1_20/v245.go create mode 100644 services/actions/schedule.go diff --git a/go.mod b/go.mod index 7404b497b7a0b..0d630ac8ec86d 100644 --- a/go.mod +++ b/go.mod @@ -91,6 +91,9 @@ require ( github.com/quasoft/websspi v1.1.2 github.com/santhosh-tekuri/jsonschema/v5 v5.2.0 github.com/sergi/go-diff v1.3.1 + github.com/robfig/cron/v3 v3.0.1 + github.com/santhosh-tekuri/jsonschema/v5 v5.1.1 + github.com/sergi/go-diff v1.2.0 github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 github.com/stretchr/testify v1.8.1 github.com/syndtr/goleveldb v1.0.0 diff --git a/go.sum b/go.sum index 3ba3fa7ab42ba..ce0a8f6c1f2c2 100644 --- a/go.sum +++ b/go.sum @@ -1055,6 +1055,8 @@ github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= diff --git a/models/actions/schedule.go b/models/actions/schedule.go new file mode 100644 index 0000000000000..843c6ab92b06e --- /dev/null +++ b/models/actions/schedule.go @@ -0,0 +1,67 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "context" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/timeutil" + webhook_module "code.gitea.io/gitea/modules/webhook" +) + +// ActionSchedule represents a schedule of a workflow file +type ActionSchedule struct { + ID int64 + Title string + Specs []string + EntryIDs []int `xorm:"entry_ids"` + RepoID int64 `xorm:"index"` + Repo *repo_model.Repository `xorm:"-"` + OwnerID int64 `xorm:"index"` + WorkflowID string `xorm:"index"` // the name of workflow file + TriggerUserID int64 + TriggerUser *user_model.User `xorm:"-"` + Ref string + CommitSHA string + Event webhook_module.HookEventType + EventPayload string `xorm:"LONGTEXT"` + Content []byte + Created timeutil.TimeStamp `xorm:"created"` + Updated timeutil.TimeStamp `xorm:"updated"` +} + +func init() { + db.RegisterModel(new(ActionSchedule)) +} + +// CreateScheduleTask creates new schedule task. +func CreateScheduleTask(ctx context.Context, id int64, rows []*ActionSchedule) error { + for _, row := range rows { + if _, err := db.GetEngine(ctx).Insert(row); err != nil { + return err + } + } + + return nil +} + +func DeleteScheduleTaskByRepo(ctx context.Context, id int64) error { + if _, err := db.GetEngine(ctx).Delete(&ActionSchedule{RepoID: id}); err != nil { + return err + } + return nil +} + +func UpdateSchedule(ctx context.Context, schedule *ActionSchedule, cols ...string) error { + sess := db.GetEngine(ctx).ID(schedule.ID) + if len(cols) > 0 { + sess.Cols(cols...) + } + _, err := sess.Update(schedule) + + return err +} diff --git a/models/actions/schedule_list.go b/models/actions/schedule_list.go new file mode 100644 index 0000000000000..6f71a3cc82738 --- /dev/null +++ b/models/actions/schedule_list.go @@ -0,0 +1,95 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "context" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/container" + + "xorm.io/builder" +) + +type ScheduleList []*ActionSchedule + +// GetUserIDs returns a slice of user's id +func (schedules ScheduleList) GetUserIDs() []int64 { + ids := make(container.Set[int64], len(schedules)) + for _, schedule := range schedules { + ids.Add(schedule.TriggerUserID) + } + return ids.Values() +} + +func (schedules ScheduleList) GetRepoIDs() []int64 { + ids := make(container.Set[int64], len(schedules)) + for _, schedule := range schedules { + ids.Add(schedule.RepoID) + } + return ids.Values() +} + +func (schedules ScheduleList) LoadTriggerUser(ctx context.Context) error { + userIDs := schedules.GetUserIDs() + users := make(map[int64]*user_model.User, len(userIDs)) + if err := db.GetEngine(ctx).In("id", userIDs).Find(&users); err != nil { + return err + } + for _, schedule := range schedules { + if schedule.TriggerUserID == user_model.ActionsUserID { + schedule.TriggerUser = user_model.NewActionsUser() + } else { + schedule.TriggerUser = users[schedule.TriggerUserID] + } + } + return nil +} + +func (schedules ScheduleList) LoadRepos() error { + repoIDs := schedules.GetRepoIDs() + repos, err := repo_model.GetRepositoriesMapByIDs(repoIDs) + if err != nil { + return err + } + for _, schedule := range schedules { + schedule.Repo = repos[schedule.RepoID] + } + return nil +} + +type FindScheduleOptions struct { + db.ListOptions + RepoID int64 + OwnerID int64 + GetAll bool +} + +func (opts FindScheduleOptions) toConds() builder.Cond { + cond := builder.NewCond() + if opts.RepoID > 0 { + cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) + } + if opts.OwnerID > 0 { + cond = cond.And(builder.Eq{"owner_id": opts.OwnerID}) + } + + return cond +} + +func FindSchedules(ctx context.Context, opts FindScheduleOptions) (ScheduleList, int64, error) { + e := db.GetEngine(ctx).Where(opts.toConds()) + if !opts.GetAll && opts.PageSize > 0 && opts.Page >= 1 { + e.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize) + } + var schedules ScheduleList + total, err := e.Desc("id").FindAndCount(&schedules) + return schedules, total, err +} + +func CountSchedules(ctx context.Context, opts FindScheduleOptions) (int64, error) { + return db.GetEngine(ctx).Where(opts.toConds()).Count(new(ActionSchedule)) +} diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 585457e474f05..175df5ee07a5b 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -467,6 +467,9 @@ var migrations = []Migration{ // v244 -> v245 NewMigration("Add NeedApproval to actions tables", v1_20.AddNeedApprovalToActionRun), + + // 245 -> 246 + NewMigration("Add action schedule table", v1_20.AddActionScheduleTable), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v1_20/v245.go b/models/migrations/v1_20/v245.go new file mode 100644 index 0000000000000..cb9713f6ee0bc --- /dev/null +++ b/models/migrations/v1_20/v245.go @@ -0,0 +1,34 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_20 //nolint + +import ( + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func AddActionScheduleTable(x *xorm.Engine) error { + type ActionSchedule struct { + ID int64 + Title string + Specs []string + EntryIDs []int `xorm:"entry_ids"` + RepoID int64 `xorm:"index"` + OwnerID int64 `xorm:"index"` + WorkflowID string `xorm:"index"` // the name of workflow file + TriggerUserID int64 + Ref string + CommitSHA string + Event string + EventPayload string `xorm:"LONGTEXT"` + Content []byte + Created timeutil.TimeStamp `xorm:"created"` + Updated timeutil.TimeStamp `xorm:"updated"` + } + + return x.Sync( + new(ActionSchedule), + ) +} diff --git a/modules/actions/workflows.go b/modules/actions/workflows.go index 7f0e6e456436b..1624ac2553e52 100644 --- a/modules/actions/workflows.go +++ b/modules/actions/workflows.go @@ -44,28 +44,41 @@ func ListWorkflows(commit *git.Commit) (git.Entries, error) { return ret, nil } -func DetectWorkflows(commit *git.Commit, triggedEvent webhook_module.HookEventType, payload api.Payloader) (map[string][]byte, error) { +func DetectWorkflows( + commit *git.Commit, + triggedEvent webhook_module.HookEventType, + payload api.Payloader, +) (map[string][]byte, map[string][]byte, error) { entries, err := ListWorkflows(commit) if err != nil { - return nil, err + return nil, nil, err } workflows := make(map[string][]byte, len(entries)) + schedules := make(map[string][]byte, len(entries)) for _, entry := range entries { f, err := entry.Blob().DataAsync() if err != nil { - return nil, err + return nil, nil, err } content, err := io.ReadAll(f) _ = f.Close() if err != nil { - return nil, err + return nil, nil, err } workflow, err := model.ReadWorkflow(bytes.NewReader(content)) if err != nil { log.Warn("ignore invalid workflow %q: %v", entry.Name(), err) continue } + + // fetch all schedule event + for _, e := range workflow.On() { + if e == "schedule" { + schedules[entry.Name()] = content + } + } + events, err := jobparser.ParseRawOn(&workflow.RawOn) if err != nil { log.Warn("ignore invalid workflow %q: %v", entry.Name(), err) @@ -81,7 +94,7 @@ func DetectWorkflows(commit *git.Commit, triggedEvent webhook_module.HookEventTy } } - return workflows, nil + return workflows, schedules, nil } func detectMatched(commit *git.Commit, triggedEvent webhook_module.HookEventType, payload api.Payloader, evt *jobparser.Event) bool { diff --git a/services/actions/init.go b/services/actions/init.go index 3fd03eeb6f26c..3f25938070c8b 100644 --- a/services/actions/init.go +++ b/services/actions/init.go @@ -19,4 +19,8 @@ func Init() { go graceful.GetManager().RunWithShutdownFns(jobEmitterQueue.Run) notification.RegisterNotifier(NewNotifier()) + + // initial all schedule task + newSchedule() + resetSchedule() } diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 574a37e9ab168..3f88415c41b65 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -4,6 +4,7 @@ package actions import ( + "bytes" "context" "fmt" "strings" @@ -24,6 +25,7 @@ import ( "code.gitea.io/gitea/services/convert" "github.com/nektos/act/pkg/jobparser" + "github.com/nektos/act/pkg/model" ) var methodCtxKey struct{} @@ -103,45 +105,135 @@ func (input *notifyInput) Notify(ctx context.Context) { } } -func notify(ctx context.Context, input *notifyInput) error { - if input.Doer.IsActions() { - // avoiding triggering cyclically, for example: - // a comment of an issue will trigger the runner to add a new comment as reply, - // and the new comment will trigger the runner again. - log.Debug("ignore executing %v for event %v whose doer is %v", getMethod(ctx), input.Event, input.Doer.Name) +func CreateScheduleTask(ctx context.Context, cron *actions_model.ActionSchedule, spec string) (int, error) { + return schedule.AddFunc(spec, func() { + run := &actions_model.ActionRun{ + Title: cron.Title, + RepoID: cron.RepoID, + OwnerID: cron.OwnerID, + WorkflowID: cron.WorkflowID, + TriggerUserID: cron.TriggerUserID, + Ref: cron.Ref, + CommitSHA: cron.CommitSHA, + Event: cron.Event, + EventPayload: cron.EventPayload, + Status: actions_model.StatusWaiting, + } + jobs, err := jobparser.Parse(cron.Content) + if err != nil { + log.Error("jobparser.Parse: %v", err) + return + } + if err := actions_model.InsertRun(ctx, run, jobs); err != nil { + log.Error("InsertRun: %v", err) + return + } + if jobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{RunID: run.ID}); err != nil { + log.Error("FindRunJobs: %v", err) + } else { + for _, job := range jobs { + if err := CreateCommitStatus(ctx, job); err != nil { + log.Error("CreateCommitStatus: %v", err) + } + } + } + }) +} + +func handleSchedules( + ctx context.Context, + schedules map[string][]byte, + commit *git.Commit, + input *notifyInput, +) error { + if commit.Branch != input.Repo.DefaultBranch { + log.Trace("commit branch is not default branch in repo") return nil } - if unit_model.TypeActions.UnitGlobalDisabled() { - return nil + + rows, _, err := actions_model.FindSchedules(ctx, actions_model.FindScheduleOptions{RepoID: input.Repo.ID}) + if err != nil { + log.Error("FindCrons: %v", err) } - if err := input.Repo.LoadUnits(ctx); err != nil { - return fmt.Errorf("repo.LoadUnits: %w", err) - } else if !input.Repo.UnitEnabled(ctx, unit_model.TypeActions) { - return nil + + for _, row := range rows { + schedule.Remove(row.EntryIDs) } - gitRepo, err := git.OpenRepository(context.Background(), input.Repo.RepoPath()) - if err != nil { - return fmt.Errorf("git.OpenRepository: %w", err) + if len(rows) > 0 { + if err := actions_model.DeleteScheduleTaskByRepo(ctx, input.Repo.ID); err != nil { + log.Error("DeleteCronTaskByRepo: %v", err) + } } - defer gitRepo.Close() - ref := input.Ref - if ref == "" { - ref = input.Repo.DefaultBranch + if len(schedules) == 0 { + log.Trace("repo %s with commit %s couldn't find schedules", input.Repo.RepoPath(), commit.ID) + return nil } - // Get the commit object for the ref - commit, err := gitRepo.GetCommit(ref) + p, err := json.Marshal(input.Payload) if err != nil { - return fmt.Errorf("gitRepo.GetCommit: %w", err) + return fmt.Errorf("json.Marshal: %w", err) } - workflows, err := actions_module.DetectWorkflows(commit, input.Event, input.Payload) - if err != nil { - return fmt.Errorf("DetectWorkflows: %w", err) + crons := make([]*actions_model.ActionSchedule, 0) + for id, content := range schedules { + log.Debug("workflow: %s, content: %v", id, string(content)) + // Check cron job condition. Only working in default branch + workflow, err := model.ReadWorkflow(bytes.NewReader(content)) + if err != nil { + log.Error("ReadWorkflow: %v", err) + continue + } + schedules := workflow.OnSchedule() + if len(schedules) == 0 { + log.Warn("no schedule event") + continue + } + log.Debug("schedules: %#v", schedules) + + crons = append(crons, &actions_model.ActionSchedule{ + Title: strings.SplitN(commit.CommitMessage, "\n", 2)[0], + RepoID: input.Repo.ID, + OwnerID: input.Repo.OwnerID, + WorkflowID: id, + TriggerUserID: input.Doer.ID, + Ref: input.Ref, + CommitSHA: commit.ID.String(), + Event: input.Event, + EventPayload: string(p), + Specs: schedules, + Content: content, + }) } + if len(crons) > 0 { + for _, cron := range crons { + entryIDs := []int{} + for _, spec := range cron.Specs { + id, err := CreateScheduleTask(ctx, cron, spec) + if err != nil { + continue + } + entryIDs = append(entryIDs, id) + } + cron.EntryIDs = entryIDs + } + + if err := actions_model.CreateScheduleTask(ctx, input.Repo.ID, crons); err != nil { + log.Error("CreateCronTask: %v", err) + } + } + + return nil +} + +func handleWorkflows( + ctx context.Context, + workflows map[string][]byte, + commit *git.Commit, + input *notifyInput, +) error { if len(workflows) == 0 { log.Trace("repo %s with commit %s couldn't find workflows", input.Repo.RepoPath(), commit.ID) return nil @@ -153,32 +245,25 @@ func notify(ctx context.Context, input *notifyInput) error { } for id, content := range workflows { - run := &actions_model.ActionRun{ + run := actions_model.ActionRun{ Title: strings.SplitN(commit.CommitMessage, "\n", 2)[0], RepoID: input.Repo.ID, OwnerID: input.Repo.OwnerID, WorkflowID: id, TriggerUserID: input.Doer.ID, - Ref: ref, + Ref: input.Ref, CommitSHA: commit.ID.String(), IsForkPullRequest: input.PullRequest != nil && input.PullRequest.IsFromFork(), Event: input.Event, EventPayload: string(p), Status: actions_model.StatusWaiting, } - if need, err := ifNeedApproval(ctx, run, input.Repo, input.Doer); err != nil { - log.Error("check if need approval for repo %d with user %d: %v", input.Repo.ID, input.Doer.ID, err) - continue - } else { - run.NeedApproval = need - } - jobs, err := jobparser.Parse(content) if err != nil { log.Error("jobparser.Parse: %v", err) continue } - if err := actions_model.InsertRun(ctx, run, jobs); err != nil { + if err := actions_model.InsertRun(ctx, &run, jobs); err != nil { log.Error("InsertRun: %v", err) continue } @@ -187,13 +272,66 @@ func notify(ctx context.Context, input *notifyInput) error { } else { for _, job := range jobs { if err := CreateCommitStatus(ctx, job); err != nil { - log.Error("Update commit status for job %v failed: %v", job.ID, err) - // go on + log.Error("CreateCommitStatus: %v", err) } } } + } + return nil +} + +func notify(ctx context.Context, input *notifyInput) error { + if input.Doer.IsActions() { + // avoiding triggering cyclically, for example: + // a comment of an issue will trigger the runner to add a new comment as reply, + // and the new comment will trigger the runner again. + log.Debug("ignore executing %v for event %v whose doer is %v", getMethod(ctx), input.Event, input.Doer.Name) + return nil + } + if unit_model.TypeActions.UnitGlobalDisabled() { + return nil + } + if err := input.Repo.LoadUnits(ctx); err != nil { + return fmt.Errorf("repo.LoadUnits: %w", err) + } else if !input.Repo.UnitEnabled(ctx, unit_model.TypeActions) { + return nil + } + gitRepo, err := git.OpenRepository(context.Background(), input.Repo.RepoPath()) + if err != nil { + return fmt.Errorf("git.OpenRepository: %w", err) } + defer gitRepo.Close() + + ref := input.Ref + if ref == "" { + ref = input.Repo.DefaultBranch + } + + // Get the commit object for the ref + commit, err := gitRepo.GetCommit(ref) + if err != nil { + return fmt.Errorf("gitRepo.GetCommit: %w", err) + } + + err = commit.LoadBranchName() + if err != nil { + return fmt.Errorf("commit.GetBranchName: %w", err) + } + + workflows, schedules, err := actions_module.DetectWorkflows(commit, input.Event, input.Payload) + if err != nil { + return fmt.Errorf("DetectWorkflows: %w", err) + } + + if err := handleSchedules(ctx, schedules, commit, input); err != nil { + log.Error("handle schedules: %v", err) + } + + if err := handleWorkflows(ctx, workflows, commit, input); err != nil { + log.Error("handle workflows: %v", err) + } + return nil } diff --git a/services/actions/schedule.go b/services/actions/schedule.go new file mode 100644 index 0000000000000..cd1ac429ad748 --- /dev/null +++ b/services/actions/schedule.go @@ -0,0 +1,63 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "context" + + actions_model "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/modules/log" + + "github.com/robfig/cron/v3" +) + +var schedule *Schedule + +type Schedule struct { + c *cron.Cron +} + +func (s *Schedule) AddFunc(spec string, f func()) (int, error) { + id, err := s.c.AddFunc(spec, f) + return int(id), err +} + +func (s *Schedule) Remove(ids []int) { + for _, id := range ids { + s.c.Remove(cron.EntryID(id)) + } +} + +func newSchedule() { + c := cron.New() + c.Start() + schedule = &Schedule{ + c: c, + } +} + +func resetSchedule() { + schedules, _, err := actions_model.FindSchedules( + context.Background(), + actions_model.FindScheduleOptions{GetAll: true}, + ) + if err != nil { + log.Error("FindSchedules: %v", err) + } + + for _, schedule := range schedules { + entryIDs := []int{} + for _, spec := range schedule.Specs { + id, err := CreateScheduleTask(context.Background(), schedule, spec) + if err != nil { + continue + } + entryIDs = append(entryIDs, id) + } + schedule.EntryIDs = entryIDs + if err := actions_model.UpdateSchedule(context.Background(), schedule, "entry_ids"); err != nil { + log.Error("UpdateSchedule: %v", err) + } + } +} From f1eb0adfa906a2699be1259c435c36ef98f824dd Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Sun, 5 Feb 2023 09:59:57 +0800 Subject: [PATCH 02/72] chore(license): add cron v3 license Signed-off-by: Bo-Yi.Wu --- assets/go-licenses.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/assets/go-licenses.json b/assets/go-licenses.json index c94d276fd6129..8ff2b9e592fff 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -864,6 +864,11 @@ "path": "github.com/robfig/cron/LICENSE", "licenseText": "Copyright (C) 2012 Rob Figueiredo\nAll Rights Reserved.\n\nMIT LICENSE\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, + { + "name": "github.com/robfig/cron/v3", + "path": "github.com/robfig/cron/v3/LICENSE", + "licenseText": "Copyright (C) 2012 Rob Figueiredo\nAll Rights Reserved.\n\nMIT LICENSE\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" + }, { "name": "github.com/rs/xid", "path": "github.com/rs/xid/LICENSE", From fffb455dd06b6e1eab07be753d87ca3afa2e7f35 Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Fri, 17 Feb 2023 05:55:32 +0800 Subject: [PATCH 03/72] refactor(actions): CreateScheduleTask --- models/actions/schedule.go | 14 ++++++++++---- services/actions/notifier_helper.go | 4 ++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/models/actions/schedule.go b/models/actions/schedule.go index 843c6ab92b06e..04738fe8aa736 100644 --- a/models/actions/schedule.go +++ b/models/actions/schedule.go @@ -39,14 +39,20 @@ func init() { } // CreateScheduleTask creates new schedule task. -func CreateScheduleTask(ctx context.Context, id int64, rows []*ActionSchedule) error { - for _, row := range rows { - if _, err := db.GetEngine(ctx).Insert(row); err != nil { +func CreateScheduleTask(ctx context.Context, rows []*ActionSchedule) error { + ctx, committer, err := db.TxContext(db.DefaultContext) + if err != nil { + return err + } + defer committer.Close() + + if len(rows) > 0 { + if err = db.Insert(ctx, rows); err != nil { return err } } - return nil + return committer.Commit() } func DeleteScheduleTaskByRepo(ctx context.Context, id int64) error { diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 3f88415c41b65..7b2dee4e01c01 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -220,8 +220,8 @@ func handleSchedules( cron.EntryIDs = entryIDs } - if err := actions_model.CreateScheduleTask(ctx, input.Repo.ID, crons); err != nil { - log.Error("CreateCronTask: %v", err) + if err := actions_model.CreateScheduleTask(ctx, crons); err != nil { + log.Error("CreateScheduleTask: %v", err) } } From 054873bbf41cccc9c6af2952da860632404e27ca Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Fri, 10 Mar 2023 18:31:44 +0800 Subject: [PATCH 04/72] chore(deps): upgrade go module Signed-off-by: Bo-Yi.Wu --- go.mod | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0d630ac8ec86d..2f264df03af4c 100644 --- a/go.mod +++ b/go.mod @@ -89,11 +89,9 @@ require ( github.com/pquerna/otp v1.4.0 github.com/prometheus/client_golang v1.14.0 github.com/quasoft/websspi v1.1.2 + github.com/robfig/cron/v3 v3.0.1 github.com/santhosh-tekuri/jsonschema/v5 v5.2.0 github.com/sergi/go-diff v1.3.1 - github.com/robfig/cron/v3 v3.0.1 - github.com/santhosh-tekuri/jsonschema/v5 v5.1.1 - github.com/sergi/go-diff v1.2.0 github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 github.com/stretchr/testify v1.8.1 github.com/syndtr/goleveldb v1.0.0 From 542479372ad49fe0c498e3d1987e613bb5a657ee Mon Sep 17 00:00:00 2001 From: Bo-Yi Wu Date: Tue, 14 Mar 2023 17:02:50 +0800 Subject: [PATCH 05/72] feat: refactor schedule tasks management in actions service - Remove `robfig/cron/v3 v3.0.1` from `go.mod` - Remove `newSchedule()` and `resetSchedule()` functions from `services/actions/init.go` - Change return type of `CreateScheduleTask()` from `(int, error)` to `error` in `services/actions/notifier_helper.go` - Add `services/actions/schedule_tasks.go` file - Add `StartScheduleTasks()` and `startTasks()` functions to `services/actions/schedule_tasks.go` - Register `start_schedule_tasks` task in `services/cron/tasks_actions.go` Signed-off-by: Bo-Yi Wu --- go.mod | 1 - go.sum | 2 - services/actions/init.go | 4 -- services/actions/notifier_helper.go | 68 ++++++++++++----------------- services/actions/schedule.go | 63 -------------------------- services/actions/schedule_tasks.go | 51 ++++++++++++++++++++++ services/cron/tasks_actions.go | 11 +++++ 7 files changed, 91 insertions(+), 109 deletions(-) delete mode 100644 services/actions/schedule.go create mode 100644 services/actions/schedule_tasks.go diff --git a/go.mod b/go.mod index 2f264df03af4c..7404b497b7a0b 100644 --- a/go.mod +++ b/go.mod @@ -89,7 +89,6 @@ require ( github.com/pquerna/otp v1.4.0 github.com/prometheus/client_golang v1.14.0 github.com/quasoft/websspi v1.1.2 - github.com/robfig/cron/v3 v3.0.1 github.com/santhosh-tekuri/jsonschema/v5 v5.2.0 github.com/sergi/go-diff v1.3.1 github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 diff --git a/go.sum b/go.sum index ce0a8f6c1f2c2..3ba3fa7ab42ba 100644 --- a/go.sum +++ b/go.sum @@ -1055,8 +1055,6 @@ github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= -github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= -github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= diff --git a/services/actions/init.go b/services/actions/init.go index 3f25938070c8b..3fd03eeb6f26c 100644 --- a/services/actions/init.go +++ b/services/actions/init.go @@ -19,8 +19,4 @@ func Init() { go graceful.GetManager().RunWithShutdownFns(jobEmitterQueue.Run) notification.RegisterNotifier(NewNotifier()) - - // initial all schedule task - newSchedule() - resetSchedule() } diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 7b2dee4e01c01..1729bbc0afec7 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -105,39 +105,36 @@ func (input *notifyInput) Notify(ctx context.Context) { } } -func CreateScheduleTask(ctx context.Context, cron *actions_model.ActionSchedule, spec string) (int, error) { - return schedule.AddFunc(spec, func() { - run := &actions_model.ActionRun{ - Title: cron.Title, - RepoID: cron.RepoID, - OwnerID: cron.OwnerID, - WorkflowID: cron.WorkflowID, - TriggerUserID: cron.TriggerUserID, - Ref: cron.Ref, - CommitSHA: cron.CommitSHA, - Event: cron.Event, - EventPayload: cron.EventPayload, - Status: actions_model.StatusWaiting, - } - jobs, err := jobparser.Parse(cron.Content) - if err != nil { - log.Error("jobparser.Parse: %v", err) - return - } - if err := actions_model.InsertRun(ctx, run, jobs); err != nil { - log.Error("InsertRun: %v", err) - return - } - if jobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{RunID: run.ID}); err != nil { - log.Error("FindRunJobs: %v", err) - } else { - for _, job := range jobs { - if err := CreateCommitStatus(ctx, job); err != nil { - log.Error("CreateCommitStatus: %v", err) - } +func CreateScheduleTask(ctx context.Context, cron *actions_model.ActionSchedule, spec string) error { + run := &actions_model.ActionRun{ + Title: cron.Title, + RepoID: cron.RepoID, + OwnerID: cron.OwnerID, + WorkflowID: cron.WorkflowID, + TriggerUserID: cron.TriggerUserID, + Ref: cron.Ref, + CommitSHA: cron.CommitSHA, + Event: cron.Event, + EventPayload: cron.EventPayload, + Status: actions_model.StatusWaiting, + } + jobs, err := jobparser.Parse(cron.Content) + if err != nil { + return err + } + if err := actions_model.InsertRun(ctx, run, jobs); err != nil { + return err + } + if jobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{RunID: run.ID}); err != nil { + return err + } else { + for _, job := range jobs { + if err := CreateCommitStatus(ctx, job); err != nil { + return err } } - }) + } + return nil } func handleSchedules( @@ -156,10 +153,6 @@ func handleSchedules( log.Error("FindCrons: %v", err) } - for _, row := range rows { - schedule.Remove(row.EntryIDs) - } - if len(rows) > 0 { if err := actions_model.DeleteScheduleTaskByRepo(ctx, input.Repo.ID); err != nil { log.Error("DeleteCronTaskByRepo: %v", err) @@ -209,15 +202,12 @@ func handleSchedules( if len(crons) > 0 { for _, cron := range crons { - entryIDs := []int{} for _, spec := range cron.Specs { - id, err := CreateScheduleTask(ctx, cron, spec) + err := CreateScheduleTask(ctx, cron, spec) if err != nil { continue } - entryIDs = append(entryIDs, id) } - cron.EntryIDs = entryIDs } if err := actions_model.CreateScheduleTask(ctx, crons); err != nil { diff --git a/services/actions/schedule.go b/services/actions/schedule.go deleted file mode 100644 index cd1ac429ad748..0000000000000 --- a/services/actions/schedule.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package actions - -import ( - "context" - - actions_model "code.gitea.io/gitea/models/actions" - "code.gitea.io/gitea/modules/log" - - "github.com/robfig/cron/v3" -) - -var schedule *Schedule - -type Schedule struct { - c *cron.Cron -} - -func (s *Schedule) AddFunc(spec string, f func()) (int, error) { - id, err := s.c.AddFunc(spec, f) - return int(id), err -} - -func (s *Schedule) Remove(ids []int) { - for _, id := range ids { - s.c.Remove(cron.EntryID(id)) - } -} - -func newSchedule() { - c := cron.New() - c.Start() - schedule = &Schedule{ - c: c, - } -} - -func resetSchedule() { - schedules, _, err := actions_model.FindSchedules( - context.Background(), - actions_model.FindScheduleOptions{GetAll: true}, - ) - if err != nil { - log.Error("FindSchedules: %v", err) - } - - for _, schedule := range schedules { - entryIDs := []int{} - for _, spec := range schedule.Specs { - id, err := CreateScheduleTask(context.Background(), schedule, spec) - if err != nil { - continue - } - entryIDs = append(entryIDs, id) - } - schedule.EntryIDs = entryIDs - if err := actions_model.UpdateSchedule(context.Background(), schedule, "entry_ids"); err != nil { - log.Error("UpdateSchedule: %v", err) - } - } -} diff --git a/services/actions/schedule_tasks.go b/services/actions/schedule_tasks.go new file mode 100644 index 0000000000000..edf69f499d5e4 --- /dev/null +++ b/services/actions/schedule_tasks.go @@ -0,0 +1,51 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "context" + "fmt" + "time" + + actions_model "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/modules/log" + + "github.com/gogs/cron" +) + +// StartScheduleTasks start the task +func StartScheduleTasks(ctx context.Context) error { + return startTasks(ctx, actions_model.FindSpecOptions{ + GetAll: true, + }) +} + +func startTasks(ctx context.Context, opts actions_model.FindSpecOptions) error { + specs, count, err := actions_model.FindSpecs(ctx, opts) + if err != nil { + return fmt.Errorf("find specs: %w", err) + } + if count == 0 { + return nil + } + + now := time.Now() + for _, row := range specs { + schedule, err := cron.Parse(row.Spec) + if err != nil { + log.Error("ParseSpec: %v", err) + continue + } + + next := schedule.Next(now) + if next.Sub(now) <= 60 { + if err := CreateScheduleTask(ctx, row.Schedule, row.Spec); err != nil { + log.Error("CreateScheduleTask: %v", err) + } + } + + } + + return nil +} diff --git a/services/cron/tasks_actions.go b/services/cron/tasks_actions.go index 30e8749a5e189..66500c11c3976 100644 --- a/services/cron/tasks_actions.go +++ b/services/cron/tasks_actions.go @@ -18,6 +18,17 @@ func initActionsTasks() { registerStopZombieTasks() registerStopEndlessTasks() registerCancelAbandonedJobs() + registerScheduleTasks() +} + +func registerScheduleTasks() { + RegisterTaskFatal("start_schedule_tasks", &BaseConfig{ + Enabled: true, + RunAtStart: true, + Schedule: "@every 1m", + }, func(ctx context.Context, _ *user_model.User, cfg Config) error { + return actions_service.StartScheduleTasks(ctx) + }) } func registerStopZombieTasks() { From dc1a6c13ef43cf59b931a56700853ff1cc56e097 Mon Sep 17 00:00:00 2001 From: Bo-Yi Wu Date: Tue, 14 Mar 2023 17:03:44 +0800 Subject: [PATCH 06/72] feat: add support for schedule specs and tasks - Add a new function `GetSchedulesMapByIDs` to retrieve schedules by ID - Create new schedule and schedule spec when creating a schedule task - Delete schedule specs when deleting a schedule - Add a new model `ActionScheduleSpec` for schedule specs Signed-off-by: Bo-Yi Wu --- models/actions/schedule.go | 41 ++++++++++--- models/actions/schedule_spec.go | 24 ++++++++ models/actions/schedule_spec_list.go | 91 ++++++++++++++++++++++++++++ models/migrations/v1_20/v245.go | 8 +++ 4 files changed, 154 insertions(+), 10 deletions(-) create mode 100644 models/actions/schedule_spec.go create mode 100644 models/actions/schedule_spec_list.go diff --git a/models/actions/schedule.go b/models/actions/schedule.go index 04738fe8aa736..cec1b2fc6b163 100644 --- a/models/actions/schedule.go +++ b/models/actions/schedule.go @@ -38,6 +38,12 @@ func init() { db.RegisterModel(new(ActionSchedule)) } +// GetSchedulesMapByIDs returns the schedules by given id slice. +func GetSchedulesMapByIDs(ids []int64) (map[int64]*ActionSchedule, error) { + schedules := make(map[int64]*ActionSchedule, len(ids)) + return schedules, db.GetEngine(db.DefaultContext).In("id", ids).Find(&schedules) +} + // CreateScheduleTask creates new schedule task. func CreateScheduleTask(ctx context.Context, rows []*ActionSchedule) error { ctx, committer, err := db.TxContext(db.DefaultContext) @@ -47,8 +53,22 @@ func CreateScheduleTask(ctx context.Context, rows []*ActionSchedule) error { defer committer.Close() if len(rows) > 0 { - if err = db.Insert(ctx, rows); err != nil { - return err + for _, row := range rows { + // create new schedule + if err = db.Insert(ctx, row); err != nil { + return err + } + + // create new schedule spec + for _, spec := range row.Specs { + if err = db.Insert(ctx, &ActionScheduleSpec{ + RepoID: row.RepoID, + ScheduleID: row.ID, + Spec: spec, + }); err != nil { + return err + } + } } } @@ -56,18 +76,19 @@ func CreateScheduleTask(ctx context.Context, rows []*ActionSchedule) error { } func DeleteScheduleTaskByRepo(ctx context.Context, id int64) error { + ctx, committer, err := db.TxContext(db.DefaultContext) + if err != nil { + return err + } + defer committer.Close() + if _, err := db.GetEngine(ctx).Delete(&ActionSchedule{RepoID: id}); err != nil { return err } - return nil -} -func UpdateSchedule(ctx context.Context, schedule *ActionSchedule, cols ...string) error { - sess := db.GetEngine(ctx).ID(schedule.ID) - if len(cols) > 0 { - sess.Cols(cols...) + if _, err := db.GetEngine(ctx).Delete(&ActionScheduleSpec{RepoID: id}); err != nil { + return err } - _, err := sess.Update(schedule) - return err + return committer.Commit() } diff --git a/models/actions/schedule_spec.go b/models/actions/schedule_spec.go new file mode 100644 index 0000000000000..ea5f97b3399d5 --- /dev/null +++ b/models/actions/schedule_spec.go @@ -0,0 +1,24 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" +) + +// ActionScheduleSpec represents a schedule spec of a workflow file +type ActionScheduleSpec struct { + ID int64 + RepoID int64 + Repo *repo_model.Repository `xorm:"-"` + ScheduleID int64 + Schedule *ActionSchedule `xorm:"-"` + + Spec string +} + +func init() { + db.RegisterModel(new(ActionScheduleSpec)) +} diff --git a/models/actions/schedule_spec_list.go b/models/actions/schedule_spec_list.go new file mode 100644 index 0000000000000..e95881e620f61 --- /dev/null +++ b/models/actions/schedule_spec_list.go @@ -0,0 +1,91 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "context" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/container" + "xorm.io/builder" +) + +type SpecList []*ActionScheduleSpec + +func (specs SpecList) GetScheduleIDs() []int64 { + ids := make(container.Set[int64], len(specs)) + for _, spec := range specs { + ids.Add(spec.ScheduleID) + } + return ids.Values() +} + +func (specs SpecList) LoadSchedules() error { + scheduleIDs := specs.GetScheduleIDs() + schedules, err := GetSchedulesMapByIDs(scheduleIDs) + if err != nil { + return err + } + for _, spec := range specs { + spec.Schedule = schedules[spec.ScheduleID] + } + return nil +} + +func (specs SpecList) GetRepoIDs() []int64 { + ids := make(container.Set[int64], len(specs)) + for _, spec := range specs { + ids.Add(spec.RepoID) + } + return ids.Values() +} + +func (specs SpecList) LoadRepos() error { + repoIDs := specs.GetRepoIDs() + repos, err := repo_model.GetRepositoriesMapByIDs(repoIDs) + if err != nil { + return err + } + for _, spec := range specs { + spec.Repo = repos[spec.RepoID] + } + return nil +} + +type FindSpecOptions struct { + db.ListOptions + RepoID int64 + GetAll bool +} + +func (opts FindSpecOptions) toConds() builder.Cond { + cond := builder.NewCond() + if opts.RepoID > 0 { + cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) + } + + return cond +} + +func FindSpecs(ctx context.Context, opts FindSpecOptions) (SpecList, int64, error) { + e := db.GetEngine(ctx).Where(opts.toConds()) + if !opts.GetAll && opts.PageSize > 0 && opts.Page >= 1 { + e.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize) + } + var specs SpecList + total, err := e.Desc("id").FindAndCount(&specs) + if err != nil { + return nil, 0, err + } + + if err := specs.LoadSchedules(); err != nil { + return nil, 0, err + } + return specs, total, nil +} + +func CountSpecs(ctx context.Context, opts FindSpecOptions) (int64, error) { + return db.GetEngine(ctx).Where(opts.toConds()).Count(new(ActionScheduleSpec)) +} diff --git a/models/migrations/v1_20/v245.go b/models/migrations/v1_20/v245.go index cb9713f6ee0bc..efeaf429c48a9 100644 --- a/models/migrations/v1_20/v245.go +++ b/models/migrations/v1_20/v245.go @@ -28,7 +28,15 @@ func AddActionScheduleTable(x *xorm.Engine) error { Updated timeutil.TimeStamp `xorm:"updated"` } + type ActionScheduleSpec struct { + ID int64 + RepoID int64 + ScheduleID int64 + Spec string + } + return x.Sync( new(ActionSchedule), + new(ActionScheduleSpec), ) } From 4fa2416299ea1f5f3489f7f79fa7d49deb97fc71 Mon Sep 17 00:00:00 2001 From: Bo-Yi Wu Date: Tue, 14 Mar 2023 17:19:42 +0800 Subject: [PATCH 07/72] refactor: refactor migration and update version number - Rename v245.go to v246.go - Add a new migration for action schedule table addition Signed-off-by: Bo-Yi Wu --- models/migrations/migrations.go | 2 +- models/migrations/v1_20/{v245.go => v246.go} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename models/migrations/v1_20/{v245.go => v246.go} (100%) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 175df5ee07a5b..a73536e62d45e 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -468,7 +468,7 @@ var migrations = []Migration{ // v244 -> v245 NewMigration("Add NeedApproval to actions tables", v1_20.AddNeedApprovalToActionRun), - // 245 -> 246 + // 246 -> 247 NewMigration("Add action schedule table", v1_20.AddActionScheduleTable), } diff --git a/models/migrations/v1_20/v245.go b/models/migrations/v1_20/v246.go similarity index 100% rename from models/migrations/v1_20/v245.go rename to models/migrations/v1_20/v246.go From 650434c03ccb7477de30c6ffd191627bd93d7a97 Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Tue, 14 Mar 2023 21:24:11 +0800 Subject: [PATCH 08/72] refactor: refactor database schema for improved organization - Rename the `Webhook org_id` column to `owner_id` - Add a new migration for creating an `action schedule` table --- models/migrations/migrations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index d126e3cca1995..6e624561bcd0f 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -469,7 +469,7 @@ var migrations = []Migration{ NewMigration("Add NeedApproval to actions tables", v1_20.AddNeedApprovalToActionRun), // v245 -> v246 NewMigration("Rename Webhook org_id to owner_id", v1_20.RenameWebhookOrgToOwner), - // 246 -> 247 + // v246 -> v247 NewMigration("Add action schedule table", v1_20.AddActionScheduleTable), } From 5a789aa628e094b02adeb1f78ddb848ec5813547 Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Tue, 14 Mar 2023 21:40:06 +0800 Subject: [PATCH 09/72] fix: refactor notifier_helper.go for improved readability and maintainability - Rename `jobs` to `workflows` in `notifier_helper.go` - Fix the `if` statement to properly continue in `notifier_helper.go` - Remove unnecessary `else` block in `notifier_helper.go` Signed-off-by: Bo-Yi.Wu --- services/actions/notifier_helper.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 1729bbc0afec7..3b903bfc3770f 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -248,22 +248,24 @@ func handleWorkflows( EventPayload: string(p), Status: actions_model.StatusWaiting, } - jobs, err := jobparser.Parse(content) + workflows, err := jobparser.Parse(content) if err != nil { log.Error("jobparser.Parse: %v", err) continue } - if err := actions_model.InsertRun(ctx, &run, jobs); err != nil { + if err := actions_model.InsertRun(ctx, &run, workflows); err != nil { log.Error("InsertRun: %v", err) continue } - if jobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{RunID: run.ID}); err != nil { + + jobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{RunID: run.ID}) + if err != nil { log.Error("FindRunJobs: %v", err) - } else { - for _, job := range jobs { - if err := CreateCommitStatus(ctx, job); err != nil { - log.Error("CreateCommitStatus: %v", err) - } + continue + } + for _, job := range jobs { + if err := CreateCommitStatus(ctx, job); err != nil { + log.Error("CreateCommitStatus: %v", err) } } } From 756c534cb104327f7e3ab69c6cc62dac4367b134 Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Tue, 14 Mar 2023 21:42:56 +0800 Subject: [PATCH 10/72] refactor: refactor `ActionSchedule` struct in schedule.go and migration v246 - Remove `EntryIDs` field from `ActionSchedule` struct in `schedule.go` - Remove `EntryIDs` field from migration v246 in `v1_20` folder Signed-off-by: Bo-Yi.Wu --- models/actions/schedule.go | 1 - models/migrations/v1_20/v246.go | 1 - 2 files changed, 2 deletions(-) diff --git a/models/actions/schedule.go b/models/actions/schedule.go index cec1b2fc6b163..bb6a536a698f7 100644 --- a/models/actions/schedule.go +++ b/models/actions/schedule.go @@ -18,7 +18,6 @@ type ActionSchedule struct { ID int64 Title string Specs []string - EntryIDs []int `xorm:"entry_ids"` RepoID int64 `xorm:"index"` Repo *repo_model.Repository `xorm:"-"` OwnerID int64 `xorm:"index"` diff --git a/models/migrations/v1_20/v246.go b/models/migrations/v1_20/v246.go index efeaf429c48a9..3ba105e2acee3 100644 --- a/models/migrations/v1_20/v246.go +++ b/models/migrations/v1_20/v246.go @@ -14,7 +14,6 @@ func AddActionScheduleTable(x *xorm.Engine) error { ID int64 Title string Specs []string - EntryIDs []int `xorm:"entry_ids"` RepoID int64 `xorm:"index"` OwnerID int64 `xorm:"index"` WorkflowID string `xorm:"index"` // the name of workflow file From 257a37f3fa492df914d4f61089fb1efa7f741a5c Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Tue, 14 Mar 2023 21:54:54 +0800 Subject: [PATCH 11/72] refactor: simplify error handling and rename 'jobs' to 'workflows' - Rename `jobs` to `workflows` for clarity - Simplify error handling in line 14 - Simplify error handling in line 19 - Remove unnecessary else statement in line 22 Signed-off-by: Bo-Yi.Wu --- services/actions/notifier_helper.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 3b903bfc3770f..b2b251d50daa7 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -118,20 +118,20 @@ func CreateScheduleTask(ctx context.Context, cron *actions_model.ActionSchedule, EventPayload: cron.EventPayload, Status: actions_model.StatusWaiting, } - jobs, err := jobparser.Parse(cron.Content) + workflows, err := jobparser.Parse(cron.Content) if err != nil { return err } - if err := actions_model.InsertRun(ctx, run, jobs); err != nil { + if err := actions_model.InsertRun(ctx, run, workflows); err != nil { return err } - if jobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{RunID: run.ID}); err != nil { + jobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{RunID: run.ID}) + if err != nil { return err - } else { - for _, job := range jobs { - if err := CreateCommitStatus(ctx, job); err != nil { - return err - } + } + for _, job := range jobs { + if err := CreateCommitStatus(ctx, job); err != nil { + return err } } return nil From a33777d32430b8981665546feaad41f160b9dd04 Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Tue, 14 Mar 2023 21:59:58 +0800 Subject: [PATCH 12/72] chore: refactor license for cron library - Remove the license for `github.com/robfig/cron/v3` Signed-off-by: Bo-Yi.Wu --- assets/go-licenses.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/assets/go-licenses.json b/assets/go-licenses.json index 8ff2b9e592fff..37093c41f1fa9 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -864,11 +864,6 @@ "path": "github.com/robfig/cron/LICENSE", "licenseText": "Copyright (C) 2012 Rob Figueiredo\nAll Rights Reserved.\n\nMIT LICENSE\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, - { - "name": "github.com/robfig/cron/v3", - "path": "github.com/robfig/cron/v3/LICENSE", - "licenseText": "Copyright (C) 2012 Rob Figueiredo\nAll Rights Reserved.\n\nMIT LICENSE\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" - }, { "name": "github.com/rs/xid", "path": "github.com/rs/xid/LICENSE", @@ -1099,4 +1094,4 @@ "path": "xorm.io/xorm/LICENSE", "licenseText": "Copyright (c) 2013 - 2015 The Xorm Authors\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n* Neither the name of the {organization} nor the names of its\n contributors may be used to endorse or promote products derived from\n this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" } -] \ No newline at end of file +] From 2f01a5181ebf19f80162886995a5c6a89745917b Mon Sep 17 00:00:00 2001 From: Bo-Yi Wu Date: Tue, 14 Mar 2023 22:07:41 +0800 Subject: [PATCH 13/72] Update go-licenses.json From a0dbb87c29ce6f33d98f34f1421ee89c082d5a23 Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Tue, 14 Mar 2023 22:09:57 +0800 Subject: [PATCH 14/72] refactor: refactor ScheduleTask functions to use passed context - Update `CreateScheduleTask` to use the passed context instead of `db.DefaultContext` - Update `DeleteScheduleTaskByRepo` to use the passed context instead of `db.DefaultContext` Signed-off-by: Bo-Yi.Wu --- models/actions/schedule.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/actions/schedule.go b/models/actions/schedule.go index bb6a536a698f7..3da152f8ef543 100644 --- a/models/actions/schedule.go +++ b/models/actions/schedule.go @@ -45,7 +45,7 @@ func GetSchedulesMapByIDs(ids []int64) (map[int64]*ActionSchedule, error) { // CreateScheduleTask creates new schedule task. func CreateScheduleTask(ctx context.Context, rows []*ActionSchedule) error { - ctx, committer, err := db.TxContext(db.DefaultContext) + ctx, committer, err := db.TxContext(ctx) if err != nil { return err } @@ -75,7 +75,7 @@ func CreateScheduleTask(ctx context.Context, rows []*ActionSchedule) error { } func DeleteScheduleTaskByRepo(ctx context.Context, id int64) error { - ctx, committer, err := db.TxContext(db.DefaultContext) + ctx, committer, err := db.TxContext(ctx) if err != nil { return err } From bf44849da5ad6793c3139bc5e2d04063c25dd7f6 Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Tue, 14 Mar 2023 22:11:16 +0800 Subject: [PATCH 15/72] refactor: remove unnecessary approval function from notifier helper file - Delete the `ifNeedApproval` function from `notifier_helper.go` Signed-off-by: Bo-Yi.Wu --- services/actions/notifier_helper.go | 37 ----------------------------- 1 file changed, 37 deletions(-) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index b2b251d50daa7..026eaa6d4cbf1 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -372,40 +372,3 @@ func notifyPackage(ctx context.Context, sender *user_model.User, pd *packages_mo }). Notify(ctx) } - -func ifNeedApproval(ctx context.Context, run *actions_model.ActionRun, repo *repo_model.Repository, user *user_model.User) (bool, error) { - // don't need approval if it's not a fork PR - if !run.IsForkPullRequest { - return false, nil - } - - // always need approval if the user is restricted - if user.IsRestricted { - log.Trace("need approval because user %d is restricted", user.ID) - return true, nil - } - - // don't need approval if the user can write - if perm, err := access_model.GetUserRepoPermission(ctx, repo, user); err != nil { - return false, fmt.Errorf("GetUserRepoPermission: %w", err) - } else if perm.CanWrite(unit_model.TypeActions) { - log.Trace("do not need approval because user %d can write", user.ID) - return false, nil - } - - // don't need approval if the user has been approved before - if count, err := actions_model.CountRuns(ctx, actions_model.FindRunOptions{ - RepoID: repo.ID, - TriggerUserID: user.ID, - Approved: true, - }); err != nil { - return false, fmt.Errorf("CountRuns: %w", err) - } else if count > 0 { - log.Trace("do not need approval because user %d has been approved before", user.ID) - return false, nil - } - - // otherwise, need approval - log.Trace("need approval because it's the first time user %d triggered actions", user.ID) - return true, nil -} From 9ca276b79e8e435813294b8f76f62b9a2b5ccafe Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Tue, 14 Mar 2023 22:21:38 +0800 Subject: [PATCH 16/72] chore: add license to go-licenses.json file - Add a license to the `go-licenses.json` file Signed-off-by: Bo-Yi.Wu --- assets/go-licenses.json | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/go-licenses.json b/assets/go-licenses.json index 37093c41f1fa9..170d3124540f6 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -1095,3 +1095,4 @@ "licenseText": "Copyright (c) 2013 - 2015 The Xorm Authors\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n* Neither the name of the {organization} nor the names of its\n contributors may be used to endorse or promote products derived from\n this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" } ] + From 4eae2df070194216d163c3a156ca6055bedccf8e Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Tue, 14 Mar 2023 22:27:52 +0800 Subject: [PATCH 17/72] style: ensure consistent line endings across all files - Remove a line that had no newline at the end of the file Signed-off-by: Bo-Yi.Wu --- assets/go-licenses.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/assets/go-licenses.json b/assets/go-licenses.json index 170d3124540f6..c94d276fd6129 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -1094,5 +1094,4 @@ "path": "xorm.io/xorm/LICENSE", "licenseText": "Copyright (c) 2013 - 2015 The Xorm Authors\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n* Neither the name of the {organization} nor the names of its\n contributors may be used to endorse or promote products derived from\n this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" } -] - +] \ No newline at end of file From 0cb21dc0bb6571dbf0c7609932ab51754aeb76a5 Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Tue, 14 Mar 2023 22:32:57 +0800 Subject: [PATCH 18/72] refactor: improve scheduling efficiency by optimizing import statements - Add an import statement in `schedule_spec_list.go` Signed-off-by: Bo-Yi.Wu --- models/actions/schedule_spec_list.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/actions/schedule_spec_list.go b/models/actions/schedule_spec_list.go index e95881e620f61..9dabe3b102825 100644 --- a/models/actions/schedule_spec_list.go +++ b/models/actions/schedule_spec_list.go @@ -9,6 +9,7 @@ import ( "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/container" + "xorm.io/builder" ) From b14c06e51d527aaead0e750dde98ed09f584b1c9 Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Fri, 17 Mar 2023 18:38:13 +0800 Subject: [PATCH 19/72] feat: refactor approval handling for workflows - Change `run` initialization to a pointer - Add function `ifNeedApproval` for deciding if a user needs approval - Modify `handleWorkflows` to use `ifNeedApproval` function - Add `NeedApproval` field to `run` in `handleWorkflows` function Signed-off-by: Bo-Yi.Wu --- services/actions/notifier_helper.go | 48 +++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 026eaa6d4cbf1..174050469b57b 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -235,7 +235,7 @@ func handleWorkflows( } for id, content := range workflows { - run := actions_model.ActionRun{ + run := &actions_model.ActionRun{ Title: strings.SplitN(commit.CommitMessage, "\n", 2)[0], RepoID: input.Repo.ID, OwnerID: input.Repo.OwnerID, @@ -248,12 +248,19 @@ func handleWorkflows( EventPayload: string(p), Status: actions_model.StatusWaiting, } + need, err := ifNeedApproval(ctx, run, input.Repo, input.Doer) + if err != nil { + log.Error("check if need approval for repo %d with user %d: %v", input.Repo.ID, input.Doer.ID, err) + continue + } + run.NeedApproval = need + workflows, err := jobparser.Parse(content) if err != nil { log.Error("jobparser.Parse: %v", err) continue } - if err := actions_model.InsertRun(ctx, &run, workflows); err != nil { + if err := actions_model.InsertRun(ctx, run, workflows); err != nil { log.Error("InsertRun: %v", err) continue } @@ -372,3 +379,40 @@ func notifyPackage(ctx context.Context, sender *user_model.User, pd *packages_mo }). Notify(ctx) } + +func ifNeedApproval(ctx context.Context, run *actions_model.ActionRun, repo *repo_model.Repository, user *user_model.User) (bool, error) { + // don't need approval if it's not a fork PR + if !run.IsForkPullRequest { + return false, nil + } + + // always need approval if the user is restricted + if user.IsRestricted { + log.Trace("need approval because user %d is restricted", user.ID) + return true, nil + } + + // don't need approval if the user can write + if perm, err := access_model.GetUserRepoPermission(ctx, repo, user); err != nil { + return false, fmt.Errorf("GetUserRepoPermission: %w", err) + } else if perm.CanWrite(unit_model.TypeActions) { + log.Trace("do not need approval because user %d can write", user.ID) + return false, nil + } + + // don't need approval if the user has been approved before + if count, err := actions_model.CountRuns(ctx, actions_model.FindRunOptions{ + RepoID: repo.ID, + TriggerUserID: user.ID, + Approved: true, + }); err != nil { + return false, fmt.Errorf("CountRuns: %w", err) + } else if count > 0 { + log.Trace("do not need approval because user %d has been approved before", user.ID) + return false, nil + } + + // otherwise, need approval + log.Trace("need approval because it's the first time user %d triggered actions", user.ID) + return true, nil +} From 5685ea075e127a08faa3fc90dd8e91bc011b4742 Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Fri, 17 Mar 2023 18:42:42 +0800 Subject: [PATCH 20/72] refactor: refactor CreateScheduleTask function - Fix a typo in the `CreateScheduleTask` function - Remove unnecessary code from the `CreateScheduleTask` function Signed-off-by: Bo-Yi.Wu --- models/actions/schedule.go | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/models/actions/schedule.go b/models/actions/schedule.go index 3da152f8ef543..65106e4f2be06 100644 --- a/models/actions/schedule.go +++ b/models/actions/schedule.go @@ -51,22 +51,24 @@ func CreateScheduleTask(ctx context.Context, rows []*ActionSchedule) error { } defer committer.Close() - if len(rows) > 0 { - for _, row := range rows { - // create new schedule - if err = db.Insert(ctx, row); err != nil { - return err - } + if len(rows) == 0 { + return nil + } - // create new schedule spec - for _, spec := range row.Specs { - if err = db.Insert(ctx, &ActionScheduleSpec{ - RepoID: row.RepoID, - ScheduleID: row.ID, - Spec: spec, - }); err != nil { - return err - } + for _, row := range rows { + // create new schedule + if err = db.Insert(ctx, row); err != nil { + return err + } + + // create new schedule spec + for _, spec := range row.Specs { + if err = db.Insert(ctx, &ActionScheduleSpec{ + RepoID: row.RepoID, + ScheduleID: row.ID, + Spec: spec, + }); err != nil { + return err } } } From 737d7fe5ac2e43171d2d8fbeaf1853f387cf26fc Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Fri, 17 Mar 2023 18:44:50 +0800 Subject: [PATCH 21/72] refactor: refactor ActionScheduleSpec model struct - Add `xorm: "index"` tag to `RepoID` and `ScheduleID` fields in `ActionScheduleSpec` - Remove `RepoID` and `ScheduleID` fields from the model struct Signed-off-by: Bo-Yi.Wu --- models/actions/schedule_spec.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/actions/schedule_spec.go b/models/actions/schedule_spec.go index ea5f97b3399d5..d4cce538a5323 100644 --- a/models/actions/schedule_spec.go +++ b/models/actions/schedule_spec.go @@ -11,10 +11,10 @@ import ( // ActionScheduleSpec represents a schedule spec of a workflow file type ActionScheduleSpec struct { ID int64 - RepoID int64 + RepoID int64 `xorm:"index"` Repo *repo_model.Repository `xorm:"-"` - ScheduleID int64 - Schedule *ActionSchedule `xorm:"-"` + ScheduleID int64 `xorm:"index"` + Schedule *ActionSchedule `xorm:"-"` Spec string } From f069367b0120a5111e4360e43d7dde0d9f8e3fdb Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Fri, 17 Mar 2023 18:48:58 +0800 Subject: [PATCH 22/72] feat: refactor schedule task creation process - Remove the function `CreateScheduleTask` from the `notifier_helper.go` file - Add the function `CreateScheduleTask` to the `schedule_tasks.go` file, which creates a scheduled task from a cron action schedule and a spec string. It creates an action run based on the schedule, inserts it into the database, and creates commit statuses for each job. Signed-off-by: Bo-Yi.Wu --- services/actions/notifier_helper.go | 32 -------------------- services/actions/schedule_tasks.go | 46 +++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 32 deletions(-) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 174050469b57b..1086f262f3e17 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -105,38 +105,6 @@ func (input *notifyInput) Notify(ctx context.Context) { } } -func CreateScheduleTask(ctx context.Context, cron *actions_model.ActionSchedule, spec string) error { - run := &actions_model.ActionRun{ - Title: cron.Title, - RepoID: cron.RepoID, - OwnerID: cron.OwnerID, - WorkflowID: cron.WorkflowID, - TriggerUserID: cron.TriggerUserID, - Ref: cron.Ref, - CommitSHA: cron.CommitSHA, - Event: cron.Event, - EventPayload: cron.EventPayload, - Status: actions_model.StatusWaiting, - } - workflows, err := jobparser.Parse(cron.Content) - if err != nil { - return err - } - if err := actions_model.InsertRun(ctx, run, workflows); err != nil { - return err - } - jobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{RunID: run.ID}) - if err != nil { - return err - } - for _, job := range jobs { - if err := CreateCommitStatus(ctx, job); err != nil { - return err - } - } - return nil -} - func handleSchedules( ctx context.Context, schedules map[string][]byte, diff --git a/services/actions/schedule_tasks.go b/services/actions/schedule_tasks.go index edf69f499d5e4..8149ecca70f88 100644 --- a/services/actions/schedule_tasks.go +++ b/services/actions/schedule_tasks.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/modules/log" "github.com/gogs/cron" + "github.com/nektos/act/pkg/jobparser" ) // StartScheduleTasks start the task @@ -49,3 +50,48 @@ func startTasks(ctx context.Context, opts actions_model.FindSpecOptions) error { return nil } + +// CreateScheduleTask creates a scheduled task from a cron action schedule and a spec string. +// It creates an action run based on the schedule, inserts it into the database, and creates commit statuses for each job. +func CreateScheduleTask(ctx context.Context, cron *actions_model.ActionSchedule, spec string) error { + // Create a new action run based on the schedule + run := &actions_model.ActionRun{ + Title: cron.Title, + RepoID: cron.RepoID, + OwnerID: cron.OwnerID, + WorkflowID: cron.WorkflowID, + TriggerUserID: cron.TriggerUserID, + Ref: cron.Ref, + CommitSHA: cron.CommitSHA, + Event: cron.Event, + EventPayload: cron.EventPayload, + Status: actions_model.StatusWaiting, + } + + // Parse the workflow specification from the cron schedule + workflows, err := jobparser.Parse(cron.Content) + if err != nil { + return err + } + + // Insert the action run and its associated jobs into the database + if err := actions_model.InsertRun(ctx, run, workflows); err != nil { + return err + } + + // Retrieve the jobs for the newly created action run + jobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{RunID: run.ID}) + if err != nil { + return err + } + + // Create commit statuses for each job + for _, job := range jobs { + if err := CreateCommitStatus(ctx, job); err != nil { + return err + } + } + + // Return nil if no errors occurred + return nil +} From 16513bb7a28cbc578b4498b779056bfd2d00cda4 Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Fri, 17 Mar 2023 19:50:24 +0800 Subject: [PATCH 23/72] refactor: refactor variable assignment in startTasks function - Change in variable assignment for `specs` in `startTasks` function Signed-off-by: Bo-Yi.Wu --- services/actions/schedule_tasks.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/services/actions/schedule_tasks.go b/services/actions/schedule_tasks.go index 8149ecca70f88..9e419995e834f 100644 --- a/services/actions/schedule_tasks.go +++ b/services/actions/schedule_tasks.go @@ -23,13 +23,10 @@ func StartScheduleTasks(ctx context.Context) error { } func startTasks(ctx context.Context, opts actions_model.FindSpecOptions) error { - specs, count, err := actions_model.FindSpecs(ctx, opts) + specs, _, err := actions_model.FindSpecs(ctx, opts) if err != nil { return fmt.Errorf("find specs: %w", err) } - if count == 0 { - return nil - } now := time.Now() for _, row := range specs { From 6c1485200eae1199889636c53fef9e62fcae1780 Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Fri, 17 Mar 2023 20:19:04 +0800 Subject: [PATCH 24/72] refactor: refactor workflow handling function signature - Remove the handleWorkflows function and replace it with a new version that has a different signature and takes more arguments. Signed-off-by: Bo-Yi.Wu --- services/actions/notifier_helper.go | 122 ++++++++++++++-------------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 1086f262f3e17..4b64bb0aa1a46 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -186,67 +186,6 @@ func handleSchedules( return nil } -func handleWorkflows( - ctx context.Context, - workflows map[string][]byte, - commit *git.Commit, - input *notifyInput, -) error { - if len(workflows) == 0 { - log.Trace("repo %s with commit %s couldn't find workflows", input.Repo.RepoPath(), commit.ID) - return nil - } - - p, err := json.Marshal(input.Payload) - if err != nil { - return fmt.Errorf("json.Marshal: %w", err) - } - - for id, content := range workflows { - run := &actions_model.ActionRun{ - Title: strings.SplitN(commit.CommitMessage, "\n", 2)[0], - RepoID: input.Repo.ID, - OwnerID: input.Repo.OwnerID, - WorkflowID: id, - TriggerUserID: input.Doer.ID, - Ref: input.Ref, - CommitSHA: commit.ID.String(), - IsForkPullRequest: input.PullRequest != nil && input.PullRequest.IsFromFork(), - Event: input.Event, - EventPayload: string(p), - Status: actions_model.StatusWaiting, - } - need, err := ifNeedApproval(ctx, run, input.Repo, input.Doer) - if err != nil { - log.Error("check if need approval for repo %d with user %d: %v", input.Repo.ID, input.Doer.ID, err) - continue - } - run.NeedApproval = need - - workflows, err := jobparser.Parse(content) - if err != nil { - log.Error("jobparser.Parse: %v", err) - continue - } - if err := actions_model.InsertRun(ctx, run, workflows); err != nil { - log.Error("InsertRun: %v", err) - continue - } - - jobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{RunID: run.ID}) - if err != nil { - log.Error("FindRunJobs: %v", err) - continue - } - for _, job := range jobs { - if err := CreateCommitStatus(ctx, job); err != nil { - log.Error("CreateCommitStatus: %v", err) - } - } - } - return nil -} - func notify(ctx context.Context, input *notifyInput) error { if input.Doer.IsActions() { // avoiding triggering cyclically, for example: @@ -384,3 +323,64 @@ func ifNeedApproval(ctx context.Context, run *actions_model.ActionRun, repo *rep log.Trace("need approval because it's the first time user %d triggered actions", user.ID) return true, nil } + +func handleWorkflows( + ctx context.Context, + workflows map[string][]byte, + commit *git.Commit, + input *notifyInput, +) error { + if len(workflows) == 0 { + log.Trace("repo %s with commit %s couldn't find workflows", input.Repo.RepoPath(), commit.ID) + return nil + } + + p, err := json.Marshal(input.Payload) + if err != nil { + return fmt.Errorf("json.Marshal: %w", err) + } + + for id, content := range workflows { + run := &actions_model.ActionRun{ + Title: strings.SplitN(commit.CommitMessage, "\n", 2)[0], + RepoID: input.Repo.ID, + OwnerID: input.Repo.OwnerID, + WorkflowID: id, + TriggerUserID: input.Doer.ID, + Ref: input.Ref, + CommitSHA: commit.ID.String(), + IsForkPullRequest: input.PullRequest != nil && input.PullRequest.IsFromFork(), + Event: input.Event, + EventPayload: string(p), + Status: actions_model.StatusWaiting, + } + need, err := ifNeedApproval(ctx, run, input.Repo, input.Doer) + if err != nil { + log.Error("check if need approval for repo %d with user %d: %v", input.Repo.ID, input.Doer.ID, err) + continue + } + run.NeedApproval = need + + workflows, err := jobparser.Parse(content) + if err != nil { + log.Error("jobparser.Parse: %v", err) + continue + } + if err := actions_model.InsertRun(ctx, run, workflows); err != nil { + log.Error("InsertRun: %v", err) + continue + } + + jobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{RunID: run.ID}) + if err != nil { + log.Error("FindRunJobs: %v", err) + continue + } + for _, job := range jobs { + if err := CreateCommitStatus(ctx, job); err != nil { + log.Error("CreateCommitStatus: %v", err) + } + } + } + return nil +} From 099d9a968eb1df288ec8832d3cc6d5dc048a5a5d Mon Sep 17 00:00:00 2001 From: Bo-Yi Wu Date: Fri, 17 Mar 2023 20:44:18 +0800 Subject: [PATCH 25/72] Update services/cron/tasks_actions.go Co-authored-by: Jason Song --- services/cron/tasks_actions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/cron/tasks_actions.go b/services/cron/tasks_actions.go index 66500c11c3976..61df52547e2cd 100644 --- a/services/cron/tasks_actions.go +++ b/services/cron/tasks_actions.go @@ -24,7 +24,7 @@ func initActionsTasks() { func registerScheduleTasks() { RegisterTaskFatal("start_schedule_tasks", &BaseConfig{ Enabled: true, - RunAtStart: true, + RunAtStart: false, Schedule: "@every 1m", }, func(ctx context.Context, _ *user_model.User, cfg Config) error { return actions_service.StartScheduleTasks(ctx) From 35081d60f191e4a312c9940b54afb910785a1e57 Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Sat, 18 Mar 2023 07:28:25 +0800 Subject: [PATCH 26/72] perf: refactor scheduling logic for more accurate timing - Update the `now` variable to truncate to the nearest minute - Change the calculation of `next` to use `now.Add(-1)` instead of `now` - Replace `next.Sub(now) <= 60` with `schedule.Next(now.Add(-1)).Equal(now)` Signed-off-by: Bo-Yi.Wu --- services/actions/schedule_tasks.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/actions/schedule_tasks.go b/services/actions/schedule_tasks.go index 9e419995e834f..558d336f886dc 100644 --- a/services/actions/schedule_tasks.go +++ b/services/actions/schedule_tasks.go @@ -28,7 +28,7 @@ func startTasks(ctx context.Context, opts actions_model.FindSpecOptions) error { return fmt.Errorf("find specs: %w", err) } - now := time.Now() + now := time.Now().Truncate(time.Minute) for _, row := range specs { schedule, err := cron.Parse(row.Spec) if err != nil { @@ -36,8 +36,7 @@ func startTasks(ctx context.Context, opts actions_model.FindSpecOptions) error { continue } - next := schedule.Next(now) - if next.Sub(now) <= 60 { + if schedule.Next(now.Add(-1)).Equal(now) { if err := CreateScheduleTask(ctx, row.Schedule, row.Spec); err != nil { log.Error("CreateScheduleTask: %v", err) } From 1655a8cbfee17689c48f2c47923a635aee8b95cb Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Sat, 18 Mar 2023 07:55:19 +0800 Subject: [PATCH 27/72] perf: improve performance and add new function for task starting - Change the initialization of `crons` to allocate capacity upfront, improving performance. - Remove an unused variable definition. - Add a new function `startTasks` with `opts` parameter. Signed-off-by: Bo-Yi.Wu --- services/actions/notifier_helper.go | 2 +- services/actions/schedule_tasks.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 4b64bb0aa1a46..785728540f719 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -137,7 +137,7 @@ func handleSchedules( return fmt.Errorf("json.Marshal: %w", err) } - crons := make([]*actions_model.ActionSchedule, 0) + crons := make([]*actions_model.ActionSchedule, 0, len(schedules)) for id, content := range schedules { log.Debug("workflow: %s, content: %v", id, string(content)) // Check cron job condition. Only working in default branch diff --git a/services/actions/schedule_tasks.go b/services/actions/schedule_tasks.go index 558d336f886dc..362b21c34ff25 100644 --- a/services/actions/schedule_tasks.go +++ b/services/actions/schedule_tasks.go @@ -41,7 +41,6 @@ func startTasks(ctx context.Context, opts actions_model.FindSpecOptions) error { log.Error("CreateScheduleTask: %v", err) } } - } return nil From bf8cda4be4e533cea9da9c26278d5928d2e942b9 Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Sat, 18 Mar 2023 09:07:52 +0800 Subject: [PATCH 28/72] refactor: simplify and optimize schedule handling function - Remove debug log for workflow content - Simplify check for empty crons slice - Remove unnecessary if statement - Simplify return statement in handleSchedules function Signed-off-by: Bo-Yi.Wu --- services/actions/notifier_helper.go | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 785728540f719..c4ec75243384f 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -139,7 +139,6 @@ func handleSchedules( crons := make([]*actions_model.ActionSchedule, 0, len(schedules)) for id, content := range schedules { - log.Debug("workflow: %s, content: %v", id, string(content)) // Check cron job condition. Only working in default branch workflow, err := model.ReadWorkflow(bytes.NewReader(content)) if err != nil { @@ -168,22 +167,20 @@ func handleSchedules( }) } - if len(crons) > 0 { - for _, cron := range crons { - for _, spec := range cron.Specs { - err := CreateScheduleTask(ctx, cron, spec) - if err != nil { - continue - } - } - } + if len(crons) == 0 { + return nil + } - if err := actions_model.CreateScheduleTask(ctx, crons); err != nil { - log.Error("CreateScheduleTask: %v", err) + for _, cron := range crons { + for _, spec := range cron.Specs { + err := CreateScheduleTask(ctx, cron, spec) + if err != nil { + continue + } } } - return nil + return actions_model.CreateScheduleTask(ctx, crons) } func notify(ctx context.Context, input *notifyInput) error { From 0b714b69ff1f366159e2b6238b5496f51341e7da Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Sat, 18 Mar 2023 11:00:15 +0800 Subject: [PATCH 29/72] refactor: refactor schedule task retrieval logic - Remove `GetAll` field from `FindSpecOptions` struct - Add `FindAllSpecs` function to retrieve all matching schedule specs - Modify `StartScheduleTasks` to use `FindAllSpecs` instead of `FindSpecs` Signed-off-by: Bo-Yi.Wu --- models/actions/schedule_spec_list.go | 42 ++++++++++++++++++++++++++-- services/actions/schedule_tasks.go | 6 ++-- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/models/actions/schedule_spec_list.go b/models/actions/schedule_spec_list.go index 9dabe3b102825..510966dc03ea7 100644 --- a/models/actions/schedule_spec_list.go +++ b/models/actions/schedule_spec_list.go @@ -58,7 +58,6 @@ func (specs SpecList) LoadRepos() error { type FindSpecOptions struct { db.ListOptions RepoID int64 - GetAll bool } func (opts FindSpecOptions) toConds() builder.Cond { @@ -70,9 +69,48 @@ func (opts FindSpecOptions) toConds() builder.Cond { return cond } +// FinAllSpecs retrieves all specs that match the given options. +// It retrieves the specs in pages of size 50 and appends them to a list until all specs have been retrieved. +// It also loads the schedules for each spec. +func FinAllSpecs(ctx context.Context, opts FindSpecOptions) (SpecList, error) { + // Set the page size and initialize the list of all specs + pageSize := 50 + var allSpecs SpecList + + // Retrieve specs in pages until all specs have been retrieved + for page := 1; ; page++ { + // Create a new query engine and apply the given conditions + e := db.GetEngine(ctx).Where(opts.toConds()) + + // Limit the results to the current page + e.Limit(pageSize, (page-1)*pageSize) + + // Retrieve the specs for the current page and add them to the list of all specs + var specs SpecList + total, err := e.Desc("id").FindAndCount(&specs) + if err != nil { + break + } + allSpecs = append(allSpecs, specs...) + + // Stop if all specs have been retrieved + if int(total) < pageSize { + break + } + } + + // Load the schedules for each spec + if err := allSpecs.LoadSchedules(); err != nil { + return nil, err + } + + // Return the list of all specs + return allSpecs, nil +} + func FindSpecs(ctx context.Context, opts FindSpecOptions) (SpecList, int64, error) { e := db.GetEngine(ctx).Where(opts.toConds()) - if !opts.GetAll && opts.PageSize > 0 && opts.Page >= 1 { + if opts.PageSize > 0 && opts.Page >= 1 { e.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize) } var specs SpecList diff --git a/services/actions/schedule_tasks.go b/services/actions/schedule_tasks.go index 362b21c34ff25..d716fc91f156b 100644 --- a/services/actions/schedule_tasks.go +++ b/services/actions/schedule_tasks.go @@ -17,13 +17,11 @@ import ( // StartScheduleTasks start the task func StartScheduleTasks(ctx context.Context) error { - return startTasks(ctx, actions_model.FindSpecOptions{ - GetAll: true, - }) + return startTasks(ctx, actions_model.FindSpecOptions{}) } func startTasks(ctx context.Context, opts actions_model.FindSpecOptions) error { - specs, _, err := actions_model.FindSpecs(ctx, opts) + specs, err := actions_model.FinAllSpecs(ctx, opts) if err != nil { return fmt.Errorf("find specs: %w", err) } From 039ead54abb5272902e05df866d8115fc1b2920d Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Sat, 18 Mar 2023 11:07:00 +0800 Subject: [PATCH 30/72] chore: refactor migration file naming convention - Rename file v246.go to v248.go in models/migrations/v1_20/ directory. Signed-off-by: Bo-Yi.Wu --- models/migrations/v1_20/{v246.go => v248.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/migrations/v1_20/{v246.go => v248.go} (100%) diff --git a/models/migrations/v1_20/v246.go b/models/migrations/v1_20/v248.go similarity index 100% rename from models/migrations/v1_20/v246.go rename to models/migrations/v1_20/v248.go From 7671a3124c588611130dff4f9b0ff0e266aa1e9e Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Sat, 18 Mar 2023 19:14:57 +0800 Subject: [PATCH 31/72] fix: refactor CreateScheduleTask function to handle empty rows gracefully - Add a check for `len(rows)` before executing `CreateScheduleTask` function - Remove a redundant check for `len(rows)` in `CreateScheduleTask` function Signed-off-by: Bo-Yi.Wu --- models/actions/schedule.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/models/actions/schedule.go b/models/actions/schedule.go index 65106e4f2be06..5e92c2a0baf1a 100644 --- a/models/actions/schedule.go +++ b/models/actions/schedule.go @@ -45,16 +45,16 @@ func GetSchedulesMapByIDs(ids []int64) (map[int64]*ActionSchedule, error) { // CreateScheduleTask creates new schedule task. func CreateScheduleTask(ctx context.Context, rows []*ActionSchedule) error { + if len(rows) == 0 { + return nil + } + ctx, committer, err := db.TxContext(ctx) if err != nil { return err } defer committer.Close() - if len(rows) == 0 { - return nil - } - for _, row := range rows { // create new schedule if err = db.Insert(ctx, row); err != nil { From fab7c2c40459d2f9ea4f33d0ae7543d4e527a98e Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Sat, 18 Mar 2023 19:48:20 +0800 Subject: [PATCH 32/72] feat: refactor schedule task creation and retrieval process - Remove the function `FinAllSpecs` from `schedule_spec_list.go` - Change the page size to 50 in `schedule_spec_list.go` - Retrieve specs in pages in `schedule_tasks.go` - Create a schedule task for each spec whose schedule is due - Check for invalid schedules before creating a schedule task Signed-off-by: Bo-Yi.Wu --- models/actions/schedule_spec_list.go | 39 ---------------------- services/actions/schedule_tasks.go | 48 ++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 52 deletions(-) diff --git a/models/actions/schedule_spec_list.go b/models/actions/schedule_spec_list.go index 510966dc03ea7..f87389b115695 100644 --- a/models/actions/schedule_spec_list.go +++ b/models/actions/schedule_spec_list.go @@ -69,45 +69,6 @@ func (opts FindSpecOptions) toConds() builder.Cond { return cond } -// FinAllSpecs retrieves all specs that match the given options. -// It retrieves the specs in pages of size 50 and appends them to a list until all specs have been retrieved. -// It also loads the schedules for each spec. -func FinAllSpecs(ctx context.Context, opts FindSpecOptions) (SpecList, error) { - // Set the page size and initialize the list of all specs - pageSize := 50 - var allSpecs SpecList - - // Retrieve specs in pages until all specs have been retrieved - for page := 1; ; page++ { - // Create a new query engine and apply the given conditions - e := db.GetEngine(ctx).Where(opts.toConds()) - - // Limit the results to the current page - e.Limit(pageSize, (page-1)*pageSize) - - // Retrieve the specs for the current page and add them to the list of all specs - var specs SpecList - total, err := e.Desc("id").FindAndCount(&specs) - if err != nil { - break - } - allSpecs = append(allSpecs, specs...) - - // Stop if all specs have been retrieved - if int(total) < pageSize { - break - } - } - - // Load the schedules for each spec - if err := allSpecs.LoadSchedules(); err != nil { - return nil, err - } - - // Return the list of all specs - return allSpecs, nil -} - func FindSpecs(ctx context.Context, opts FindSpecOptions) (SpecList, int64, error) { e := db.GetEngine(ctx).Where(opts.toConds()) if opts.PageSize > 0 && opts.Page >= 1 { diff --git a/services/actions/schedule_tasks.go b/services/actions/schedule_tasks.go index d716fc91f156b..e9c3efa703f70 100644 --- a/services/actions/schedule_tasks.go +++ b/services/actions/schedule_tasks.go @@ -9,6 +9,7 @@ import ( "time" actions_model "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/log" "github.com/gogs/cron" @@ -20,27 +21,48 @@ func StartScheduleTasks(ctx context.Context) error { return startTasks(ctx, actions_model.FindSpecOptions{}) } +// startTasks retrieves all specs in pages of size 50 and creates a schedule task for each spec +// whose schedule is due at the current minute. func startTasks(ctx context.Context, opts actions_model.FindSpecOptions) error { - specs, err := actions_model.FinAllSpecs(ctx, opts) - if err != nil { - return fmt.Errorf("find specs: %w", err) - } + // Set the page size + pageSize := 50 - now := time.Now().Truncate(time.Minute) - for _, row := range specs { - schedule, err := cron.Parse(row.Spec) + // Retrieve specs in pages until all specs have been retrieved + for page := 1; ; page++ { + // Retrieve the specs for the current page + specs, total, err := actions_model.FindSpecs(ctx, actions_model.FindSpecOptions{ + ListOptions: db.ListOptions{ + Page: page, + PageSize: pageSize, + }, + }) if err != nil { - log.Error("ParseSpec: %v", err) - continue + return fmt.Errorf("find specs: %w", err) } - if schedule.Next(now.Add(-1)).Equal(now) { - if err := CreateScheduleTask(ctx, row.Schedule, row.Spec); err != nil { - log.Error("CreateScheduleTask: %v", err) + // Check if the schedule for each spec is due at the current minute + now := time.Now().Truncate(time.Minute) + for _, row := range specs { + schedule, err := cron.Parse(row.Spec) + if err != nil { + // Skip specs with invalid schedules + log.Error("ParseSpec: %v", err) + continue + } + + // Create a schedule task for specs whose schedule is due at the current minute + if schedule.Next(now.Add(-1)).Equal(now) { + if err := CreateScheduleTask(ctx, row.Schedule, row.Spec); err != nil { + log.Error("CreateScheduleTask: %v", err) + } } } - } + // Stop if all specs have been retrieved + if int(total) < pageSize { + break + } + } return nil } From b815dacf758e2cfa0f86fd6929e4a5c4b447a858 Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Sat, 18 Mar 2023 19:49:53 +0800 Subject: [PATCH 33/72] chore: replace space with tab Signed-off-by: Bo-Yi.Wu --- models/migrations/migrations.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 21d12f5d65691..43a69b81f4aa7 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -473,8 +473,8 @@ var migrations = []Migration{ NewMigration("Add missed column owner_id for project table", v1_20.AddNewColumnForProject), // v247 -> v248 NewMigration("Fix incorrect project type", v1_20.FixIncorrectProjectType), - // v248 -> v249 - NewMigration("Add action schedule table", v1_20.AddActionScheduleTable), + // v248 -> v249 + NewMigration("Add action schedule table", v1_20.AddActionScheduleTable), } // GetCurrentDBVersion returns the current db version From 978739dae7e0b88e26bfcde6eb836e5ed138be5c Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Sat, 18 Mar 2023 20:17:12 +0800 Subject: [PATCH 34/72] refactor: refactor schedule task registration function - Remove `registerScheduleTasks` function - Add a new implementation of `registerScheduleTasks` function Signed-off-by: Bo-Yi.Wu --- services/cron/tasks_actions.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/services/cron/tasks_actions.go b/services/cron/tasks_actions.go index 61df52547e2cd..0875792503d52 100644 --- a/services/cron/tasks_actions.go +++ b/services/cron/tasks_actions.go @@ -21,16 +21,6 @@ func initActionsTasks() { registerScheduleTasks() } -func registerScheduleTasks() { - RegisterTaskFatal("start_schedule_tasks", &BaseConfig{ - Enabled: true, - RunAtStart: false, - Schedule: "@every 1m", - }, func(ctx context.Context, _ *user_model.User, cfg Config) error { - return actions_service.StartScheduleTasks(ctx) - }) -} - func registerStopZombieTasks() { RegisterTaskFatal("stop_zombie_tasks", &BaseConfig{ Enabled: true, @@ -60,3 +50,16 @@ func registerCancelAbandonedJobs() { return actions_service.CancelAbandonedJobs(ctx) }) } + +// registerScheduleTasks registers a scheduled task that runs every minute to start any due schedule tasks. +func registerScheduleTasks() { + // Register the task with a unique name, enabled status, and schedule for every minute. + RegisterTaskFatal("start_schedule_tasks", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@every 1m", + }, func(ctx context.Context, _ *user_model.User, cfg Config) error { + // Call the function to start schedule tasks and pass the context. + return actions_service.StartScheduleTasks(ctx) + }) +} From 081569689c5c0520dc99a4465ff86fcdc1cce86b Mon Sep 17 00:00:00 2001 From: Bo-Yi Wu Date: Mon, 20 Mar 2023 14:27:12 +0800 Subject: [PATCH 35/72] refactor: refactor file naming convention - File `v248.go` was renamed to `v249.go`. Signed-off-by: Bo-Yi Wu --- models/migrations/v1_20/{v248.go => v249.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/migrations/v1_20/{v248.go => v249.go} (100%) diff --git a/models/migrations/v1_20/v248.go b/models/migrations/v1_20/v249.go similarity index 100% rename from models/migrations/v1_20/v248.go rename to models/migrations/v1_20/v249.go From a693fe6da05e08143cf291d1460ac5ce038f5526 Mon Sep 17 00:00:00 2001 From: Bo-Yi Wu Date: Mon, 20 Mar 2023 14:30:05 +0800 Subject: [PATCH 36/72] chore: convert space to tab --- models/migrations/migrations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 62bed7c53b54e..68686107260bf 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -475,7 +475,7 @@ var migrations = []Migration{ NewMigration("Fix incorrect project type", v1_20.FixIncorrectProjectType), // v248 -> v249 NewMigration("Add version column to action_runner table", v1_20.AddVersionToActionRunner), - // v249 -> v250 + // v249 -> v250 NewMigration("Add action schedule table", v1_20.AddActionScheduleTable), } From 5d0e0e6d41a59cd671bae756cd2d49d0eb6801e2 Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Sat, 25 Mar 2023 12:09:35 +0800 Subject: [PATCH 37/72] chore: refactor migration file names in v1_20 directory - Rename `v249.go` to `v250.go` in `models/migrations/v1_20` Signed-off-by: Bo-Yi.Wu --- models/migrations/v1_20/{v249.go => v250.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/migrations/v1_20/{v249.go => v250.go} (100%) diff --git a/models/migrations/v1_20/v249.go b/models/migrations/v1_20/v250.go similarity index 100% rename from models/migrations/v1_20/v249.go rename to models/migrations/v1_20/v250.go From ce80ce8b097bc4174c312dcc78dcd82510ac280f Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Sun, 26 Mar 2023 08:37:57 +0800 Subject: [PATCH 38/72] N/A (no changes in git diff): refactor code for improved performance and readability No summary comments, as there are no changes in the git diff. Signed-off-by: Bo-Yi.Wu --- models/migrations/migrations.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 1af09ab8c643e..9accedd0e482f 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -477,8 +477,8 @@ var migrations = []Migration{ NewMigration("Add version column to action_runner table", v1_20.AddVersionToActionRunner), // v249 -> v250 NewMigration("Improve Action table indices v3", v1_20.ImproveActionTableIndices), - // v250 -> v251 - NewMigration("Add action schedule table", v1_20.AddActionScheduleTable), + // v250 -> v251 + NewMigration("Add action schedule table", v1_20.AddActionScheduleTable), } // GetCurrentDBVersion returns the current db version From 958e67bde6a955991c2b1e022107b9c2ac32184d Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Fri, 31 Mar 2023 21:13:49 +0800 Subject: [PATCH 39/72] chore: update Signed-off-by: Bo-Yi.Wu --- services/actions/notifier_helper.go | 2 +- services/actions/schedule_tasks.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index c4ec75243384f..b4e54bb17e738 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -374,7 +374,7 @@ func handleWorkflows( continue } for _, job := range jobs { - if err := CreateCommitStatus(ctx, job); err != nil { + if err := createCommitStatus(ctx, job); err != nil { log.Error("CreateCommitStatus: %v", err) } } diff --git a/services/actions/schedule_tasks.go b/services/actions/schedule_tasks.go index e9c3efa703f70..d93be5a221579 100644 --- a/services/actions/schedule_tasks.go +++ b/services/actions/schedule_tasks.go @@ -102,7 +102,7 @@ func CreateScheduleTask(ctx context.Context, cron *actions_model.ActionSchedule, // Create commit statuses for each job for _, job := range jobs { - if err := CreateCommitStatus(ctx, job); err != nil { + if err := createCommitStatus(ctx, job); err != nil { return err } } From ad91c4b65dbb345b9d998cf30db7c3f6be732cff Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Fri, 31 Mar 2023 21:49:50 +0800 Subject: [PATCH 40/72] chore: update Signed-off-by: Bo-Yi.Wu --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1ee95b9f59063..2d25487df26df 100644 --- a/go.mod +++ b/go.mod @@ -289,7 +289,7 @@ replace github.com/shurcooL/vfsgen => github.com/lunny/vfsgen v0.0.0-20220105142 replace github.com/blevesearch/zapx/v15 v15.3.6 => github.com/zeripath/zapx/v15 v15.3.6-alignment-fix -replace github.com/nektos/act => gitea.com/gitea/act v0.243.2-0.20230329055922-5e76853b55ab +replace github.com/nektos/act => gitea.com/gitea/act v0.243.2 exclude github.com/gofrs/uuid v3.2.0+incompatible diff --git a/go.sum b/go.sum index 6bbbbf6eea086..5724622e5ca52 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,8 @@ codeberg.org/gusted/mcaptcha v0.0.0-20220723083913-4f3072e1d570/go.mod h1:IIAjsi dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078 h1:cliQ4HHsCo6xi2oWZYKWW4bly/Ory9FuTpFPRxj/mAg= git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078/go.mod h1:g/V2Hjas6Z1UHUp4yIx6bATpNzJ7DYtD0FG3+xARWxs= -gitea.com/gitea/act v0.243.2-0.20230329055922-5e76853b55ab h1:HDImhO/XpMJrw2PJcADI/wgur9Gro/pegLFaRt8Wpg0= -gitea.com/gitea/act v0.243.2-0.20230329055922-5e76853b55ab/go.mod h1:mabw6AZAiDgxGlK83orWLrNERSPvgBJzEUS3S7u2bHI= +gitea.com/gitea/act v0.243.2 h1:4fbAbHKpVzj2NV3RDB6auO5hsmpLdLixsgLCHys7cPs= +gitea.com/gitea/act v0.243.2/go.mod h1:mabw6AZAiDgxGlK83orWLrNERSPvgBJzEUS3S7u2bHI= gitea.com/go-chi/binding v0.0.0-20221013104517-b29891619681 h1:MMSPgnVULVwV9kEBgvyEUhC9v/uviZ55hPJEMjpbNR4= gitea.com/go-chi/binding v0.0.0-20221013104517-b29891619681/go.mod h1:77TZu701zMXWJFvB8gvTbQ92zQ3DQq/H7l5wAEjQRKc= gitea.com/go-chi/cache v0.0.0-20210110083709-82c4c9ce2d5e/go.mod h1:k2V/gPDEtXGjjMGuBJiapffAXTv76H4snSmlJRLUhH0= From 513a4bf7cc7a32fbdd37ff9a844a74e09a0fc75b Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Tue, 11 Apr 2023 21:21:54 +0800 Subject: [PATCH 41/72] chore: rename Signed-off-by: Bo-Yi.Wu --- models/migrations/v1_20/{v250.go => v251.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/migrations/v1_20/{v250.go => v251.go} (100%) diff --git a/models/migrations/v1_20/v250.go b/models/migrations/v1_20/v251.go similarity index 100% rename from models/migrations/v1_20/v250.go rename to models/migrations/v1_20/v251.go From e5257edfb66c16c221575c54470c18b229b2b1bf Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Tue, 11 Apr 2023 21:22:43 +0800 Subject: [PATCH 42/72] chore: rename Signed-off-by: Bo-Yi.Wu --- models/migrations/v1_20/{v251.go => v252.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/migrations/v1_20/{v251.go => v252.go} (100%) diff --git a/models/migrations/v1_20/v251.go b/models/migrations/v1_20/v252.go similarity index 100% rename from models/migrations/v1_20/v251.go rename to models/migrations/v1_20/v252.go From 83a5fb530275948fc28132a3b7261487d36f96ee Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Tue, 11 Apr 2023 21:38:38 +0800 Subject: [PATCH 43/72] refactor: simplify schedule event detection - Remove code that fetched all schedule events and only keep the schedule event detection - Simplify the schedule event detection code Signed-off-by: Bo-Yi.Wu --- modules/actions/workflows.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/modules/actions/workflows.go b/modules/actions/workflows.go index 340f57ea0a429..848ef7ceecc0a 100644 --- a/modules/actions/workflows.go +++ b/modules/actions/workflows.go @@ -98,18 +98,6 @@ func DetectWorkflows( if err != nil { return nil, nil, err } - workflow, err := model.ReadWorkflow(bytes.NewReader(content)) - if err != nil { - log.Warn("ignore invalid workflow %q: %v", entry.Name(), err) - continue - } - - // fetch all schedule event - for _, e := range workflow.On() { - if e == "schedule" { - schedules[entry.Name()] = content - } - } events, err := GetEventsFromContent(content) if err != nil { @@ -118,6 +106,9 @@ func DetectWorkflows( } for _, evt := range events { log.Trace("detect workflow %q for event %#v matching %q", entry.Name(), evt, triggedEvent) + if evt.IsSchedule() { + schedules[entry.Name()] = content + } if detectMatched(commit, triggedEvent, payload, evt) { workflows[entry.Name()] = content } From 4db836d83fb0ca663d1f1727bd56471f5675057a Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Tue, 11 Apr 2023 21:39:17 +0800 Subject: [PATCH 44/72] chore: update Signed-off-by: Bo-Yi.Wu --- models/migrations/migrations.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 3a0526586c624..0c2e1290da09e 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -481,8 +481,8 @@ var migrations = []Migration{ NewMigration("Change Container Metadata", v1_20.ChangeContainerMetadataMultiArch), // v251 -> v252 NewMigration("Fix incorrect owner team unit access mode", v1_20.FixIncorrectOwnerTeamUnitAccessMode), - // v252 -> v253 - NewMigration("Add action schedule table", v1_20.AddActionScheduleTable), + // v252 -> v253 + NewMigration("Add action schedule table", v1_20.AddActionScheduleTable), } // GetCurrentDBVersion returns the current db version From 606027fd16579fc001301f36e060c16aaaf7ab1f Mon Sep 17 00:00:00 2001 From: Bo-Yi Wu Date: Thu, 13 Apr 2023 08:43:55 +0800 Subject: [PATCH 45/72] Update services/actions/schedule_tasks.go Co-authored-by: Lunny Xiao --- services/actions/schedule_tasks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/actions/schedule_tasks.go b/services/actions/schedule_tasks.go index d93be5a221579..c91a1449ee0e7 100644 --- a/services/actions/schedule_tasks.go +++ b/services/actions/schedule_tasks.go @@ -59,7 +59,7 @@ func startTasks(ctx context.Context, opts actions_model.FindSpecOptions) error { } // Stop if all specs have been retrieved - if int(total) < pageSize { + if len(specs) < pageSize { break } } From f972c37ef69ffeb412ce92b9309c146638032b56 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Mon, 17 Apr 2023 11:57:09 -0400 Subject: [PATCH 46/72] bump migration name --- models/migrations/v1_20/{v252.go => v254.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/migrations/v1_20/{v252.go => v254.go} (100%) diff --git a/models/migrations/v1_20/v252.go b/models/migrations/v1_20/v254.go similarity index 100% rename from models/migrations/v1_20/v252.go rename to models/migrations/v1_20/v254.go From 6c7170cd7a6ddf0edb01d3798b6a4596747afab2 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Mon, 17 Apr 2023 12:06:17 -0400 Subject: [PATCH 47/72] fix syntax issue --- services/actions/schedule_tasks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/actions/schedule_tasks.go b/services/actions/schedule_tasks.go index c91a1449ee0e7..2314b38738e86 100644 --- a/services/actions/schedule_tasks.go +++ b/services/actions/schedule_tasks.go @@ -30,7 +30,7 @@ func startTasks(ctx context.Context, opts actions_model.FindSpecOptions) error { // Retrieve specs in pages until all specs have been retrieved for page := 1; ; page++ { // Retrieve the specs for the current page - specs, total, err := actions_model.FindSpecs(ctx, actions_model.FindSpecOptions{ + specs, _, err := actions_model.FindSpecs(ctx, actions_model.FindSpecOptions{ ListOptions: db.ListOptions{ Page: page, PageSize: pageSize, From a2fade1c795e8781a595f00650ac1bb73af47666 Mon Sep 17 00:00:00 2001 From: appleboy Date: Sat, 29 Apr 2023 11:18:10 +0800 Subject: [PATCH 48/72] chore: update Signed-off-by: appleboy --- models/migrations/v1_20/{v254.go => v256.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/migrations/v1_20/{v254.go => v256.go} (100%) diff --git a/models/migrations/v1_20/v254.go b/models/migrations/v1_20/v256.go similarity index 100% rename from models/migrations/v1_20/v254.go rename to models/migrations/v1_20/v256.go From c3564116a52a16132a64e0057414893b0f9c1c0f Mon Sep 17 00:00:00 2001 From: appleboy Date: Sat, 29 Apr 2023 11:20:06 +0800 Subject: [PATCH 49/72] chore: update Signed-off-by: appleboy --- models/migrations/migrations.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 211b352a54652..ef9af7107281b 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -486,6 +486,10 @@ var migrations = []Migration{ // v253 -> v254 NewMigration("Fix ExternalTracker and ExternalWiki accessMode in owner and admin team", v1_20.FixExternalTrackerAndExternalWikiAccessModeInOwnerAndAdminTeam), // v254 -> v255 + NewMigration("Add ActionTaskOutput table", v1_20.AddActionTaskOutputTable), + // v255 -> v256 + NewMigration("Add ArchivedUnix Column", v1_20.AddArchivedUnixToRepository), + // v256 -> v257 NewMigration("Add action schedule table", v1_20.AddActionScheduleTable), } From d9589433cd77b1e245e5aebe6d28fd7ed583e345 Mon Sep 17 00:00:00 2001 From: appleboy Date: Sat, 29 Apr 2023 15:12:20 +0800 Subject: [PATCH 50/72] refactor: optimize scheduling and improve API usage - Remove loop that creates schedule tasks for each cron and spec Signed-off-by: appleboy --- services/actions/notifier_helper.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 04adab8a9fbd7..9671db8262e9f 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -171,15 +171,6 @@ func handleSchedules( return nil } - for _, cron := range crons { - for _, spec := range cron.Specs { - err := CreateScheduleTask(ctx, cron, spec) - if err != nil { - continue - } - } - } - return actions_model.CreateScheduleTask(ctx, crons) } From 7c8b678aa2ca92fcb1e9ce3a2309f4d513b2db4f Mon Sep 17 00:00:00 2001 From: appleboy Date: Sat, 29 Apr 2023 16:45:31 +0800 Subject: [PATCH 51/72] chore: add dashboard.start_schedule_tasks language Signed-off-by: appleboy --- options/locale/locale_en-US.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 0072ac6fc36fa..947dd4fc1ad2f 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2644,6 +2644,7 @@ dashboard.gc_lfs = Garbage collect LFS meta objects dashboard.stop_zombie_tasks = Stop zombie tasks dashboard.stop_endless_tasks = Stop endless tasks dashboard.cancel_abandoned_jobs = Cancel abandoned jobs +dashboard.start_schedule_tasks = Start schedule tasks users.user_manage_panel = User Account Management users.new_account = Create User Account From c7f89f93223429023d7ead86b3308a1cb094c034 Mon Sep 17 00:00:00 2001 From: appleboy Date: Sat, 29 Apr 2023 22:14:54 +0800 Subject: [PATCH 52/72] chore: update Signed-off-by: appleboy --- services/actions/schedule_tasks.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/actions/schedule_tasks.go b/services/actions/schedule_tasks.go index 2314b38738e86..ee8db8ff21c34 100644 --- a/services/actions/schedule_tasks.go +++ b/services/actions/schedule_tasks.go @@ -18,12 +18,12 @@ import ( // StartScheduleTasks start the task func StartScheduleTasks(ctx context.Context) error { - return startTasks(ctx, actions_model.FindSpecOptions{}) + return startTasks(ctx) } // startTasks retrieves all specs in pages of size 50 and creates a schedule task for each spec // whose schedule is due at the current minute. -func startTasks(ctx context.Context, opts actions_model.FindSpecOptions) error { +func startTasks(ctx context.Context) error { // Set the page size pageSize := 50 From 67e2da62b901e803bba8b1c34f4d86a5c460a568 Mon Sep 17 00:00:00 2001 From: appleboy Date: Sun, 30 Apr 2023 18:55:17 +0800 Subject: [PATCH 53/72] feat: improve scheduling and task handling in Gogs API - Import "time" and "github.com/gogs/cron" packages - Add GetReposMapByIDs function to retrieve repos by given ID slice - Add comments and refactor CreateScheduleTask function - Import "context" and "github.com/gogs/cron" in schedule_spec.go - Add new fields and Parse function to ActionScheduleSpec struct - Add UpdateScheduleSpec function - Update LoadSchedules function in schedule_spec_list.go - Add Next field in FindSpecOptions struct - Update models/migrations/v1_20/v256.go - Refactor startTasks function in schedule_tasks.go Signed-off-by: appleboy --- models/actions/schedule.go | 26 +++++++++++++++++-- models/actions/schedule_spec.go | 26 +++++++++++++++++++ models/actions/schedule_spec_list.go | 15 +++++++++++ models/migrations/v1_20/v256.go | 5 ++++ services/actions/schedule_tasks.go | 37 +++++++++++++++++----------- 5 files changed, 92 insertions(+), 17 deletions(-) diff --git a/models/actions/schedule.go b/models/actions/schedule.go index 5e92c2a0baf1a..52ce801d9152f 100644 --- a/models/actions/schedule.go +++ b/models/actions/schedule.go @@ -5,12 +5,15 @@ package actions import ( "context" + "time" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/timeutil" webhook_module "code.gitea.io/gitea/modules/webhook" + + "github.com/gogs/cron" ) // ActionSchedule represents a schedule of a workflow file @@ -43,36 +46,55 @@ func GetSchedulesMapByIDs(ids []int64) (map[int64]*ActionSchedule, error) { return schedules, db.GetEngine(db.DefaultContext).In("id", ids).Find(&schedules) } +// GetReposMapByIDs returns the repos by given id slice. +func GetReposMapByIDs(ids []int64) (map[int64]*repo_model.Repository, error) { + repos := make(map[int64]*repo_model.Repository, len(ids)) + return repos, db.GetEngine(db.DefaultContext).In("id", ids).Find(&repos) +} + // CreateScheduleTask creates new schedule task. func CreateScheduleTask(ctx context.Context, rows []*ActionSchedule) error { + // Return early if there are no rows to insert if len(rows) == 0 { return nil } + // Begin transaction ctx, committer, err := db.TxContext(ctx) if err != nil { return err } defer committer.Close() + // Loop through each schedule row for _, row := range rows { - // create new schedule + // Create new schedule row if err = db.Insert(ctx, row); err != nil { return err } - // create new schedule spec + // Loop through each schedule spec and create a new spec row + now := time.Now() for _, spec := range row.Specs { + // Parse the spec and check for errors + schedule, err := cron.Parse(spec) + if err != nil { + continue // skip to the next spec if there's an error + } + + // Insert the new schedule spec row if err = db.Insert(ctx, &ActionScheduleSpec{ RepoID: row.RepoID, ScheduleID: row.ID, Spec: spec, + Next: timeutil.TimeStamp(schedule.Next(now).Unix()), }); err != nil { return err } } } + // Commit transaction return committer.Commit() } diff --git a/models/actions/schedule_spec.go b/models/actions/schedule_spec.go index d4cce538a5323..d214ade5fc9cc 100644 --- a/models/actions/schedule_spec.go +++ b/models/actions/schedule_spec.go @@ -4,8 +4,13 @@ package actions import ( + "context" + "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/timeutil" + + "github.com/gogs/cron" ) // ActionScheduleSpec represents a schedule spec of a workflow file @@ -16,9 +21,30 @@ type ActionScheduleSpec struct { ScheduleID int64 `xorm:"index"` Schedule *ActionSchedule `xorm:"-"` + // Next time the job will run, or the zero time if Cron has not been + // started or this entry's schedule is unsatisfiable + Next timeutil.TimeStamp `xorm:"index"` + // Prev is the last time this job was run, or the zero time if never. + Prev timeutil.TimeStamp Spec string + + Created timeutil.TimeStamp `xorm:"created"` + Updated timeutil.TimeStamp `xorm:"updated"` +} + +func (s *ActionScheduleSpec) Parse() (cron.Schedule, error) { + return cron.Parse(s.Spec) } func init() { db.RegisterModel(new(ActionScheduleSpec)) } + +func UpdateScheduleSpec(ctx context.Context, spec *ActionScheduleSpec, cols ...string) error { + sess := db.GetEngine(ctx).ID(spec.ID) + if len(cols) > 0 { + sess.Cols(cols...) + } + _, err := sess.Update(spec) + return err +} diff --git a/models/actions/schedule_spec_list.go b/models/actions/schedule_spec_list.go index f87389b115695..d379490b4e91c 100644 --- a/models/actions/schedule_spec_list.go +++ b/models/actions/schedule_spec_list.go @@ -32,6 +32,16 @@ func (specs SpecList) LoadSchedules() error { for _, spec := range specs { spec.Schedule = schedules[spec.ScheduleID] } + + repoIDs := specs.GetRepoIDs() + repos, err := GetReposMapByIDs(repoIDs) + if err != nil { + return err + } + for _, spec := range specs { + spec.Repo = repos[spec.RepoID] + } + return nil } @@ -58,6 +68,7 @@ func (specs SpecList) LoadRepos() error { type FindSpecOptions struct { db.ListOptions RepoID int64 + Next int64 } func (opts FindSpecOptions) toConds() builder.Cond { @@ -66,6 +77,10 @@ func (opts FindSpecOptions) toConds() builder.Cond { cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) } + if opts.Next > 0 { + cond = cond.And(builder.Lte{"next": opts.Next}) + } + return cond } diff --git a/models/migrations/v1_20/v256.go b/models/migrations/v1_20/v256.go index 3ba105e2acee3..e49348fb1cbb3 100644 --- a/models/migrations/v1_20/v256.go +++ b/models/migrations/v1_20/v256.go @@ -32,6 +32,11 @@ func AddActionScheduleTable(x *xorm.Engine) error { RepoID int64 ScheduleID int64 Spec string + Next timeutil.TimeStamp `xorm:"index"` + Prev timeutil.TimeStamp + + Created timeutil.TimeStamp `xorm:"created"` + Updated timeutil.TimeStamp `xorm:"updated"` } return x.Sync( diff --git a/services/actions/schedule_tasks.go b/services/actions/schedule_tasks.go index ee8db8ff21c34..997ae75dfafb2 100644 --- a/services/actions/schedule_tasks.go +++ b/services/actions/schedule_tasks.go @@ -11,8 +11,8 @@ import ( actions_model "code.gitea.io/gitea/models/actions" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" - "github.com/gogs/cron" "github.com/nektos/act/pkg/jobparser" ) @@ -21,13 +21,15 @@ func StartScheduleTasks(ctx context.Context) error { return startTasks(ctx) } -// startTasks retrieves all specs in pages of size 50 and creates a schedule task for each spec -// whose schedule is due at the current minute. +// startTasks retrieves specifications in pages, creates a schedule task for each specification, +// and updates the specification's next run time and previous run time. +// The function returns an error if there's an issue with finding or updating the specifications. func startTasks(ctx context.Context) error { // Set the page size pageSize := 50 // Retrieve specs in pages until all specs have been retrieved + now := time.Now() for page := 1; ; page++ { // Retrieve the specs for the current page specs, _, err := actions_model.FindSpecs(ctx, actions_model.FindSpecOptions{ @@ -35,26 +37,30 @@ func startTasks(ctx context.Context) error { Page: page, PageSize: pageSize, }, + Next: now.Unix(), }) if err != nil { return fmt.Errorf("find specs: %w", err) } - // Check if the schedule for each spec is due at the current minute - now := time.Now().Truncate(time.Minute) + // Loop through each spec and create a schedule task for it for _, row := range specs { - schedule, err := cron.Parse(row.Spec) + if err := CreateScheduleTask(ctx, row.Schedule); err != nil { + log.Error("CreateScheduleTask: %v", err) + } + + // Parse the spec + schedule, err := row.Parse() if err != nil { - // Skip specs with invalid schedules - log.Error("ParseSpec: %v", err) + log.Error("Parse: %v", err) continue } - // Create a schedule task for specs whose schedule is due at the current minute - if schedule.Next(now.Add(-1)).Equal(now) { - if err := CreateScheduleTask(ctx, row.Schedule, row.Spec); err != nil { - log.Error("CreateScheduleTask: %v", err) - } + // Update the spec's next run time and previous run time + row.Prev = row.Next + row.Next = timeutil.TimeStamp(schedule.Next(now.Add(1 * time.Minute)).Unix()) + if err := actions_model.UpdateScheduleSpec(ctx, row, "prev", "next"); err != nil { + log.Error("UpdateScheduleSpec: %v", err) } } @@ -63,12 +69,13 @@ func startTasks(ctx context.Context) error { break } } + return nil } -// CreateScheduleTask creates a scheduled task from a cron action schedule and a spec string. +// CreateScheduleTask creates a scheduled task from a cron action schedule. // It creates an action run based on the schedule, inserts it into the database, and creates commit statuses for each job. -func CreateScheduleTask(ctx context.Context, cron *actions_model.ActionSchedule, spec string) error { +func CreateScheduleTask(ctx context.Context, cron *actions_model.ActionSchedule) error { // Create a new action run based on the schedule run := &actions_model.ActionRun{ Title: cron.Title, From 06fc4315fb9809301bb941981db90468f0af0a46 Mon Sep 17 00:00:00 2001 From: appleboy Date: Sun, 30 Apr 2023 19:23:31 +0800 Subject: [PATCH 54/72] feat: improve API efficiency and test coverage - Add ActionScheduleSpec and ActionSchedule with RepoID in DeleteRepository function Signed-off-by: appleboy --- models/repo.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/repo.go b/models/repo.go index 5a1e2e028e81c..0175ccaaa2870 100644 --- a/models/repo.go +++ b/models/repo.go @@ -165,6 +165,8 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error { &actions_model.ActionRunJob{RepoID: repoID}, &actions_model.ActionRun{RepoID: repoID}, &actions_model.ActionRunner{RepoID: repoID}, + &actions_model.ActionScheduleSpec{RepoID: repoID}, + &actions_model.ActionSchedule{RepoID: repoID}, ); err != nil { return fmt.Errorf("deleteBeans: %w", err) } From f2c002130cbffe72158c0d72696c6fc094450ae0 Mon Sep 17 00:00:00 2001 From: appleboy Date: Sat, 6 May 2023 15:58:20 +0800 Subject: [PATCH 55/72] fix: improve error handling and API efficiency in tests - Add a missing error return statement in `handleSchedules` function Signed-off-by: appleboy --- services/actions/notifier_helper.go | 1 + 1 file changed, 1 insertion(+) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 9671db8262e9f..af3c3c4463cdc 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -119,6 +119,7 @@ func handleSchedules( rows, _, err := actions_model.FindSchedules(ctx, actions_model.FindScheduleOptions{RepoID: input.Repo.ID}) if err != nil { log.Error("FindCrons: %v", err) + return err } if len(rows) > 0 { From 5c9f14fa9491716453e651551f68acd8b4bedec9 Mon Sep 17 00:00:00 2001 From: appleboy Date: Sat, 6 May 2023 16:01:02 +0800 Subject: [PATCH 56/72] refactor: improve action scheduling and API usage - Remove `index` tag from `WorkflowID` in `ActionSchedule` struct - Add `index` tag to `RepoID` and `ScheduleID` in `AddActionScheduleTable` function Signed-off-by: appleboy --- models/actions/schedule.go | 2 +- models/migrations/v1_20/v256.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/models/actions/schedule.go b/models/actions/schedule.go index 52ce801d9152f..64a1ce290ebea 100644 --- a/models/actions/schedule.go +++ b/models/actions/schedule.go @@ -24,7 +24,7 @@ type ActionSchedule struct { RepoID int64 `xorm:"index"` Repo *repo_model.Repository `xorm:"-"` OwnerID int64 `xorm:"index"` - WorkflowID string `xorm:"index"` // the name of workflow file + WorkflowID string TriggerUserID int64 TriggerUser *user_model.User `xorm:"-"` Ref string diff --git a/models/migrations/v1_20/v256.go b/models/migrations/v1_20/v256.go index e49348fb1cbb3..a80ad99326d35 100644 --- a/models/migrations/v1_20/v256.go +++ b/models/migrations/v1_20/v256.go @@ -14,9 +14,9 @@ func AddActionScheduleTable(x *xorm.Engine) error { ID int64 Title string Specs []string - RepoID int64 `xorm:"index"` - OwnerID int64 `xorm:"index"` - WorkflowID string `xorm:"index"` // the name of workflow file + RepoID int64 `xorm:"index"` + OwnerID int64 `xorm:"index"` + WorkflowID string TriggerUserID int64 Ref string CommitSHA string @@ -29,8 +29,8 @@ func AddActionScheduleTable(x *xorm.Engine) error { type ActionScheduleSpec struct { ID int64 - RepoID int64 - ScheduleID int64 + RepoID int64 `xorm:"index"` + ScheduleID int64 `xorm:"index"` Spec string Next timeutil.TimeStamp `xorm:"index"` Prev timeutil.TimeStamp From 6c93d4b3aeaeddaba3b6c33be8ff5e69d2249096 Mon Sep 17 00:00:00 2001 From: appleboy Date: Wed, 10 May 2023 22:12:45 +0800 Subject: [PATCH 57/72] refactor: improve API efficiency and test robustness - Rename migration file from `v256.go` to `v257.go` Signed-off-by: appleboy --- models/migrations/v1_20/{v256.go => v257.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/migrations/v1_20/{v256.go => v257.go} (100%) diff --git a/models/migrations/v1_20/v256.go b/models/migrations/v1_20/v257.go similarity index 100% rename from models/migrations/v1_20/v256.go rename to models/migrations/v1_20/v257.go From 4c243edc69a383521b81da0c9caf242a5d9f5df7 Mon Sep 17 00:00:00 2001 From: appleboy Date: Wed, 10 May 2023 22:14:25 +0800 Subject: [PATCH 58/72] chore: update Signed-off-by: appleboy --- models/migrations/migrations.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 6cd0c02b5b475..3b715912e60c1 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -491,8 +491,8 @@ var migrations = []Migration{ NewMigration("Add ArchivedUnix Column", v1_20.AddArchivedUnixToRepository), // v256 -> v257 NewMigration("Add is_internal column to package", v1_20.AddIsInternalColumnToPackage), - // v257 -> v258 - NewMigration("Add action schedule table", v1_20.AddActionScheduleTable), + // v257 -> v258 + NewMigration("Add action schedule table", v1_20.AddActionScheduleTable), } // GetCurrentDBVersion returns the current db version From 7ad692f1b48159d104747bff2321cdc0a6191b15 Mon Sep 17 00:00:00 2001 From: Bo-Yi Wu Date: Tue, 25 Jul 2023 11:26:36 +0800 Subject: [PATCH 59/72] rename Signed-off-by: Bo-Yi Wu --- models/migrations/migrations.go | 2 -- models/migrations/{v1_20/v257.go => v1_21/v270.go} | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) rename models/migrations/{v1_20/v257.go => v1_21/v270.go} (97%) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 3b715912e60c1..0e84ae9f0e29a 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -491,8 +491,6 @@ var migrations = []Migration{ NewMigration("Add ArchivedUnix Column", v1_20.AddArchivedUnixToRepository), // v256 -> v257 NewMigration("Add is_internal column to package", v1_20.AddIsInternalColumnToPackage), - // v257 -> v258 - NewMigration("Add action schedule table", v1_20.AddActionScheduleTable), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v1_20/v257.go b/models/migrations/v1_21/v270.go similarity index 97% rename from models/migrations/v1_20/v257.go rename to models/migrations/v1_21/v270.go index a80ad99326d35..2c045f2a0ec8f 100644 --- a/models/migrations/v1_20/v257.go +++ b/models/migrations/v1_21/v270.go @@ -1,7 +1,7 @@ // Copyright 2023 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package v1_20 //nolint +package v1_21 //nolint import ( "code.gitea.io/gitea/modules/timeutil" From 36c58fe39b883e177da1ca66587a7ef193a7a0d2 Mon Sep 17 00:00:00 2001 From: Bo-Yi Wu Date: Tue, 25 Jul 2023 16:42:55 +0800 Subject: [PATCH 60/72] rename Signed-off-by: Bo-Yi Wu --- models/migrations/migrations.go | 2 ++ models/migrations/v1_21/{v270.go => v269.go} | 0 2 files changed, 2 insertions(+) rename models/migrations/v1_21/{v270.go => v269.go} (100%) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index fc11d54071fa4..eeaf83d92b8ff 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -519,6 +519,8 @@ var migrations = []Migration{ NewMigration("Add action_tasks_version table", v1_21.CreateActionTasksVersionTable), // v268 -> v269 NewMigration("Update Action Ref", v1_21.UpdateActionsRefIndex), + // v269 -> v270 + NewMigration("Add Action Schedule Table", v1_21.AddActionScheduleTable), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v1_21/v270.go b/models/migrations/v1_21/v269.go similarity index 100% rename from models/migrations/v1_21/v270.go rename to models/migrations/v1_21/v269.go From 729bc69c0e97be161964cea6f6b844c9b557769e Mon Sep 17 00:00:00 2001 From: Bo-Yi Wu Date: Sun, 13 Aug 2023 17:05:15 +0800 Subject: [PATCH 61/72] rename Signed-off-by: Bo-Yi Wu --- models/migrations/v1_21/{v269.go => v270.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/migrations/v1_21/{v269.go => v270.go} (100%) diff --git a/models/migrations/v1_21/v269.go b/models/migrations/v1_21/v270.go similarity index 100% rename from models/migrations/v1_21/v269.go rename to models/migrations/v1_21/v270.go From c1be2b5657ae9d41ed3ff693fbafc6f2b7d9181b Mon Sep 17 00:00:00 2001 From: Bo-Yi Wu Date: Sun, 13 Aug 2023 17:06:25 +0800 Subject: [PATCH 62/72] rename Signed-off-by: Bo-Yi Wu --- models/migrations/v1_21/{v270.go => v271.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/migrations/v1_21/{v270.go => v271.go} (100%) diff --git a/models/migrations/v1_21/v270.go b/models/migrations/v1_21/v271.go similarity index 100% rename from models/migrations/v1_21/v270.go rename to models/migrations/v1_21/v271.go From 40ed679855b11833ecdd8eb3b704d3d68ca7bcba Mon Sep 17 00:00:00 2001 From: appleboy Date: Sun, 13 Aug 2023 19:25:08 +0800 Subject: [PATCH 63/72] feat: refactor schedule handling in notifier helper - Add `github.com/gogs/cron` to the `go.mod` file - Remove `GetAll` from `FindScheduleOptions` struct in `schedule_list.go` - Replace `if !opts.GetAll` with `if !opts.ListAll` in `FindSchedules` function in `schedule_list.go` - Remove `handleSchedules` function from `notifier_helper.go` - Modify `notify` function in `notifier_helper.go` to include `schedules` in `DetectWorkflows` function calls - Replace `workflows` with `detectedWorkflows` in `handleWorkflows` function in `notifier_helper.go` - Add `handleSchedules` function in `notifier_helper.go` to handle detected workflows and create schedule tasks - Modify `handleWorkflows` function in `notifier_helper.go` to cancel running jobs if the event is push - Modify `notify` function in `notifier_helper.go` to call `handleWorkflows` with `detectedWorkflows` instead of `workflows` Signed-off-by: appleboy --- go.mod | 1 + go.sum | 2 + models/actions/schedule_list.go | 3 +- services/actions/notifier_helper.go | 258 ++++++++++++---------------- 4 files changed, 112 insertions(+), 152 deletions(-) diff --git a/go.mod b/go.mod index b1e31c3faaff9..4602d88e8f017 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( github.com/go-webauthn/webauthn v0.8.6 github.com/gobwas/glob v0.2.3 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f + github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14 github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/go-github/v53 v53.2.0 diff --git a/go.sum b/go.sum index 5e942457a5571..8d5002f9aaefe 100644 --- a/go.sum +++ b/go.sum @@ -452,6 +452,8 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= +github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14 h1:yXtpJr/LV6PFu4nTLgfjQdcMdzjbqqXMEnHfq0Or6p8= +github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14/go.mod h1:jPoNZLWDAqA5N3G5amEoiNbhVrmM+ZQEcnQvNQ2KaZk= github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 h1:UjoPNDAQ5JPCjlxoJd6K8ALZqSDDhk2ymieAZOVaDg0= github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85/go.mod h1:fR6z1Ie6rtF7kl/vBYMfgD5/G5B1blui7z426/sj2DU= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= diff --git a/models/actions/schedule_list.go b/models/actions/schedule_list.go index 6f71a3cc82738..e873c05ec3179 100644 --- a/models/actions/schedule_list.go +++ b/models/actions/schedule_list.go @@ -65,7 +65,6 @@ type FindScheduleOptions struct { db.ListOptions RepoID int64 OwnerID int64 - GetAll bool } func (opts FindScheduleOptions) toConds() builder.Cond { @@ -82,7 +81,7 @@ func (opts FindScheduleOptions) toConds() builder.Cond { func FindSchedules(ctx context.Context, opts FindScheduleOptions) (ScheduleList, int64, error) { e := db.GetEngine(ctx).Where(opts.toConds()) - if !opts.GetAll && opts.PageSize > 0 && opts.Page >= 1 { + if !opts.ListAll && opts.PageSize > 0 && opts.Page >= 1 { e.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize) } var schedules ScheduleList diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 05171734b38cd..291f024b16073 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -105,76 +105,6 @@ func (input *notifyInput) Notify(ctx context.Context) { } } -func handleSchedules( - ctx context.Context, - schedules map[string][]byte, - commit *git.Commit, - input *notifyInput, -) error { - if commit.Branch != input.Repo.DefaultBranch { - log.Trace("commit branch is not default branch in repo") - return nil - } - - rows, _, err := actions_model.FindSchedules(ctx, actions_model.FindScheduleOptions{RepoID: input.Repo.ID}) - if err != nil { - log.Error("FindCrons: %v", err) - return err - } - - if len(rows) > 0 { - if err := actions_model.DeleteScheduleTaskByRepo(ctx, input.Repo.ID); err != nil { - log.Error("DeleteCronTaskByRepo: %v", err) - } - } - - if len(schedules) == 0 { - log.Trace("repo %s with commit %s couldn't find schedules", input.Repo.RepoPath(), commit.ID) - return nil - } - - p, err := json.Marshal(input.Payload) - if err != nil { - return fmt.Errorf("json.Marshal: %w", err) - } - - crons := make([]*actions_model.ActionSchedule, 0, len(schedules)) - for id, content := range schedules { - // Check cron job condition. Only working in default branch - workflow, err := model.ReadWorkflow(bytes.NewReader(content)) - if err != nil { - log.Error("ReadWorkflow: %v", err) - continue - } - schedules := workflow.OnSchedule() - if len(schedules) == 0 { - log.Warn("no schedule event") - continue - } - log.Debug("schedules: %#v", schedules) - - crons = append(crons, &actions_model.ActionSchedule{ - Title: strings.SplitN(commit.CommitMessage, "\n", 2)[0], - RepoID: input.Repo.ID, - OwnerID: input.Repo.OwnerID, - WorkflowID: id, - TriggerUserID: input.Doer.ID, - Ref: input.Ref, - CommitSHA: commit.ID.String(), - Event: input.Event, - EventPayload: string(p), - Specs: schedules, - Content: content, - }) - } - - if len(crons) == 0 { - return nil - } - - return actions_model.CreateScheduleTask(ctx, crons) -} - func notify(ctx context.Context, input *notifyInput) error { if input.Doer.IsActions() { // avoiding triggering cyclically, for example: @@ -214,13 +144,8 @@ func notify(ctx context.Context, input *notifyInput) error { return fmt.Errorf("gitRepo.GetCommit: %w", err) } - err = commit.LoadBranchName() - if err != nil { - return fmt.Errorf("commit.GetBranchName: %w", err) - } - var detectedWorkflows []*actions_module.DetectedWorkflow - workflows, err := actions_module.DetectWorkflows(gitRepo, commit, input.Event, input.Payload) + workflows, schedules, err := actions_module.DetectWorkflows(gitRepo, commit, input.Event, input.Payload) if err != nil { return fmt.Errorf("DetectWorkflows: %w", err) } @@ -241,7 +166,7 @@ func notify(ctx context.Context, input *notifyInput) error { if err != nil { return fmt.Errorf("gitRepo.GetCommit: %w", err) } - baseWorkflows, err := actions_module.DetectWorkflows(gitRepo, baseCommit, input.Event, input.Payload) + baseWorkflows, _, err := actions_module.DetectWorkflows(gitRepo, baseCommit, input.Event, input.Payload) if err != nil { return fmt.Errorf("DetectWorkflows: %w", err) } @@ -256,76 +181,12 @@ func notify(ctx context.Context, input *notifyInput) error { } } - if len(detectedWorkflows) == 0 { - return nil - } - - workflows, schedules, err := actions_module.DetectWorkflows(commit, input.Event, input.Payload) - if err != nil { - return fmt.Errorf("DetectWorkflows: %w", err) - } - if err := handleSchedules(ctx, schedules, commit, input); err != nil { log.Error("handle schedules: %v", err) } - - if err := handleWorkflows(ctx, workflows, commit, input); err != nil { + if err := handleWorkflows(ctx, detectedWorkflows, commit, input); err != nil { log.Error("handle workflows: %v", err) - } - - for _, dwf := range detectedWorkflows { - run := &actions_model.ActionRun{ - Title: strings.SplitN(commit.CommitMessage, "\n", 2)[0], - RepoID: input.Repo.ID, - OwnerID: input.Repo.OwnerID, - WorkflowID: dwf.EntryName, - TriggerUserID: input.Doer.ID, - Ref: ref, - CommitSHA: commit.ID.String(), - IsForkPullRequest: isForkPullRequest, - Event: input.Event, - EventPayload: string(p), - TriggerEvent: dwf.TriggerEvent, - Status: actions_model.StatusWaiting, - } - if need, err := ifNeedApproval(ctx, run, input.Repo, input.Doer); err != nil { - log.Error("check if need approval for repo %d with user %d: %v", input.Repo.ID, input.Doer.ID, err) - continue - } else { - run.NeedApproval = need - } - - jobs, err := jobparser.Parse(dwf.Content) - if err != nil { - log.Error("jobparser.Parse: %v", err) - continue - } - - // cancel running jobs if the event is push - if run.Event == webhook_module.HookEventPush { - // cancel running jobs of the same workflow - if err := actions_model.CancelRunningJobs( - ctx, - run.RepoID, - run.Ref, - run.WorkflowID, - ); err != nil { - log.Error("CancelRunningJobs: %v", err) - } - } - - if err := actions_model.InsertRun(ctx, run, jobs); err != nil { - log.Error("InsertRun: %v", err) - continue - } - - alljobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{RunID: run.ID}) - if err != nil { - log.Error("FindRunJobs: %v", err) - continue - } - CreateCommitStatus(ctx, alljobs...) } return nil @@ -418,11 +279,11 @@ func ifNeedApproval(ctx context.Context, run *actions_model.ActionRun, repo *rep func handleWorkflows( ctx context.Context, - workflows map[string][]byte, + detectedWorkflows []*actions_module.DetectedWorkflow, commit *git.Commit, input *notifyInput, ) error { - if len(workflows) == 0 { + if len(detectedWorkflows) == 0 { log.Trace("repo %s with commit %s couldn't find workflows", input.Repo.RepoPath(), commit.ID) return nil } @@ -447,12 +308,12 @@ func handleWorkflows( } } - for id, content := range workflows { + for _, dwf := range detectedWorkflows { run := &actions_model.ActionRun{ Title: strings.SplitN(commit.CommitMessage, "\n", 2)[0], RepoID: input.Repo.ID, OwnerID: input.Repo.OwnerID, - WorkflowID: id, + WorkflowID: dwf.EntryName, TriggerUserID: input.Doer.ID, Ref: input.Ref, CommitSHA: commit.ID.String(), @@ -468,22 +329,36 @@ func handleWorkflows( } run.NeedApproval = need - workflows, err := jobparser.Parse(content) + jobs, err := jobparser.Parse(dwf.Content) if err != nil { log.Error("jobparser.Parse: %v", err) continue } - if err := actions_model.InsertRun(ctx, run, workflows); err != nil { + + // cancel running jobs if the event is push + if run.Event == webhook_module.HookEventPush { + // cancel running jobs of the same workflow + if err := actions_model.CancelRunningJobs( + ctx, + run.RepoID, + run.Ref, + run.WorkflowID, + ); err != nil { + log.Error("CancelRunningJobs: %v", err) + } + } + + if err := actions_model.InsertRun(ctx, run, jobs); err != nil { log.Error("InsertRun: %v", err) continue } - jobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{RunID: run.ID}) + alljobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{RunID: run.ID}) if err != nil { log.Error("FindRunJobs: %v", err) continue } - for _, job := range jobs { + for _, job := range alljobs { if err := createCommitStatus(ctx, job); err != nil { log.Error("CreateCommitStatus: %v", err) } @@ -491,3 +366,86 @@ func handleWorkflows( } return nil } + +func handleSchedules( + ctx context.Context, + detectedWorkflows []*actions_module.DetectedWorkflow, + commit *git.Commit, + input *notifyInput, +) error { + if len(detectedWorkflows) == 0 { + log.Trace("repo %s with commit %s couldn't find schedules", input.Repo.RepoPath(), commit.ID) + return nil + } + + branch, err := commit.GetBranchName() + if err != nil { + return err + } + if branch != input.Repo.DefaultBranch { + log.Trace("commit branch is not default branch in repo") + return nil + } + + rows, _, err := actions_model.FindSchedules(ctx, actions_model.FindScheduleOptions{RepoID: input.Repo.ID}) + if err != nil { + log.Error("FindCrons: %v", err) + return err + } + + if len(rows) > 0 { + if err := actions_model.DeleteScheduleTaskByRepo(ctx, input.Repo.ID); err != nil { + log.Error("DeleteCronTaskByRepo: %v", err) + } + } + + p, err := json.Marshal(input.Payload) + if err != nil { + return fmt.Errorf("json.Marshal: %w", err) + } + + crons := make([]*actions_model.ActionSchedule, 0, len(detectedWorkflows)) + for _, dwf := range detectedWorkflows { + // Check cron job condition. Only working in default branch + workflow, err := model.ReadWorkflow(bytes.NewReader(dwf.Content)) + if err != nil { + log.Error("ReadWorkflow: %v", err) + continue + } + schedules := workflow.OnSchedule() + if len(schedules) == 0 { + log.Warn("no schedule event") + continue + } + + run := &actions_model.ActionSchedule{ + Title: strings.SplitN(commit.CommitMessage, "\n", 2)[0], + RepoID: input.Repo.ID, + OwnerID: input.Repo.OwnerID, + WorkflowID: dwf.EntryName, + TriggerUserID: input.Doer.ID, + Ref: input.Ref, + CommitSHA: commit.ID.String(), + Event: input.Event, + EventPayload: string(p), + Specs: schedules, + Content: dwf.Content, + } + + // cancel running jobs if the event is push + if run.Event == webhook_module.HookEventPush { + // cancel running jobs of the same workflow + if err := actions_model.CancelRunningJobs( + ctx, + run.RepoID, + run.Ref, + run.WorkflowID, + ); err != nil { + log.Error("CancelRunningJobs: %v", err) + } + } + crons = append(crons) + } + + return actions_model.CreateScheduleTask(ctx, crons) +} From a12d22dc9053917529370ff3954a66cc3c73869a Mon Sep 17 00:00:00 2001 From: appleboy Date: Sun, 13 Aug 2023 19:29:18 +0800 Subject: [PATCH 64/72] fix: improve error handling in `startTasks` function - Add error return statements in the `startTasks` function in two places. Signed-off-by: appleboy --- services/actions/schedule_tasks.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/actions/schedule_tasks.go b/services/actions/schedule_tasks.go index 997ae75dfafb2..91d932406c05c 100644 --- a/services/actions/schedule_tasks.go +++ b/services/actions/schedule_tasks.go @@ -47,6 +47,7 @@ func startTasks(ctx context.Context) error { for _, row := range specs { if err := CreateScheduleTask(ctx, row.Schedule); err != nil { log.Error("CreateScheduleTask: %v", err) + return err } // Parse the spec @@ -61,6 +62,7 @@ func startTasks(ctx context.Context) error { row.Next = timeutil.TimeStamp(schedule.Next(now.Add(1 * time.Minute)).Unix()) if err := actions_model.UpdateScheduleSpec(ctx, row, "prev", "next"); err != nil { log.Error("UpdateScheduleSpec: %v", err) + return err } } From 7890d91ca9f184039bdf49129c8121021e637137 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 22 Aug 2023 16:02:39 +0800 Subject: [PATCH 65/72] remove gogs/cron --- go.mod | 3 +-- go.sum | 2 -- models/actions/schedule.go | 5 +++-- models/actions/schedule_spec.go | 5 +++-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 63c4fb867ae12..1996d29a84bd1 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,6 @@ require ( github.com/go-webauthn/webauthn v0.8.6 github.com/gobwas/glob v0.2.3 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f - github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14 github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/go-github/v53 v53.2.0 @@ -91,6 +90,7 @@ require ( github.com/prometheus/client_golang v1.16.0 github.com/quasoft/websspi v1.1.2 github.com/redis/go-redis/v9 v9.0.5 + github.com/robfig/cron/v3 v3.0.1 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sassoftware/go-rpmutils v0.2.0 github.com/sergi/go-diff v1.3.1 @@ -255,7 +255,6 @@ require ( github.com/rhysd/actionlint v1.6.25 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/robfig/cron v1.2.0 // indirect - github.com/robfig/cron/v3 v3.0.1 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/rs/xid v1.5.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect diff --git a/go.sum b/go.sum index aae9d6ea43670..4b92b1ee637d1 100644 --- a/go.sum +++ b/go.sum @@ -455,8 +455,6 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= -github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14 h1:yXtpJr/LV6PFu4nTLgfjQdcMdzjbqqXMEnHfq0Or6p8= -github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14/go.mod h1:jPoNZLWDAqA5N3G5amEoiNbhVrmM+ZQEcnQvNQ2KaZk= github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 h1:UjoPNDAQ5JPCjlxoJd6K8ALZqSDDhk2ymieAZOVaDg0= github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85/go.mod h1:fR6z1Ie6rtF7kl/vBYMfgD5/G5B1blui7z426/sj2DU= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= diff --git a/models/actions/schedule.go b/models/actions/schedule.go index 64a1ce290ebea..07a4a0f5ce5d7 100644 --- a/models/actions/schedule.go +++ b/models/actions/schedule.go @@ -13,7 +13,7 @@ import ( "code.gitea.io/gitea/modules/timeutil" webhook_module "code.gitea.io/gitea/modules/webhook" - "github.com/gogs/cron" + "github.com/robfig/cron/v3" ) // ActionSchedule represents a schedule of a workflow file @@ -75,9 +75,10 @@ func CreateScheduleTask(ctx context.Context, rows []*ActionSchedule) error { // Loop through each schedule spec and create a new spec row now := time.Now() + p := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor) for _, spec := range row.Specs { // Parse the spec and check for errors - schedule, err := cron.Parse(spec) + schedule, err := p.Parse(spec) if err != nil { continue // skip to the next spec if there's an error } diff --git a/models/actions/schedule_spec.go b/models/actions/schedule_spec.go index d214ade5fc9cc..6d7876b1ae87a 100644 --- a/models/actions/schedule_spec.go +++ b/models/actions/schedule_spec.go @@ -10,7 +10,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/timeutil" - "github.com/gogs/cron" + "github.com/robfig/cron/v3" ) // ActionScheduleSpec represents a schedule spec of a workflow file @@ -33,7 +33,8 @@ type ActionScheduleSpec struct { } func (s *ActionScheduleSpec) Parse() (cron.Schedule, error) { - return cron.Parse(s.Spec) + p := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor) + return p.Parse(s.Spec) } func init() { From c40bc37f9a4119f554de76db01fa237c07852cd6 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 22 Aug 2023 16:31:01 +0800 Subject: [PATCH 66/72] Fix bug --- services/actions/notifier_helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index f4f4a63d2d9bc..5e5f40e728c04 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -451,7 +451,7 @@ func handleSchedules( log.Error("CancelRunningJobs: %v", err) } } - crons = append(crons) + crons = append(crons, run) } return actions_model.CreateScheduleTask(ctx, crons) From dd27c7fb693133a95d6bd6e2b230d69b6669279a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 22 Aug 2023 16:53:48 +0800 Subject: [PATCH 67/72] Fix bug --- modules/actions/workflows.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/actions/workflows.go b/modules/actions/workflows.go index 5d4c33110d705..408fdb8f8ef22 100644 --- a/modules/actions/workflows.go +++ b/modules/actions/workflows.go @@ -106,8 +106,8 @@ func DetectWorkflows( return nil, nil, err } - workflows := make([]*DetectedWorkflow, len(entries)) - schedules := make([]*DetectedWorkflow, len(entries)) + workflows := make([]*DetectedWorkflow, 0, len(entries)) + schedules := make([]*DetectedWorkflow, 0, len(entries)) for _, entry := range entries { content, err := GetContentFromEntry(entry) if err != nil { From 82694d757d3d9ddda7764db758145558ef154a27 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 22 Aug 2023 23:05:11 +0800 Subject: [PATCH 68/72] Fix bug --- services/actions/notifier_helper.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 5e5f40e728c04..1d1ba8fdade31 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -145,15 +145,15 @@ func notify(ctx context.Context, input *notifyInput) error { } var detectedWorkflows []*actions_module.DetectedWorkflow + actionsConfig := input.Repo.MustGetUnit(ctx, unit_model.TypeActions).ActionsConfig() workflows, schedules, err := actions_module.DetectWorkflows(gitRepo, commit, input.Event, input.Payload) if err != nil { return fmt.Errorf("DetectWorkflows: %w", err) } + if len(workflows) == 0 { log.Trace("repo %s with commit %s couldn't find workflows", input.Repo.RepoPath(), commit.ID) } else { - actionsConfig := input.Repo.MustGetUnit(ctx, unit_model.TypeActions).ActionsConfig() - for _, wf := range workflows { if actionsConfig.IsWorkflowDisabled(wf.EntryName) { log.Trace("repo %s has disable workflows %s", input.Repo.RepoPath(), wf.EntryName) @@ -192,7 +192,7 @@ func notify(ctx context.Context, input *notifyInput) error { log.Error("handle schedules: %v", err) } - if err := handleWorkflows(ctx, detectedWorkflows, commit, input); err != nil { + if err := handleWorkflows(ctx, detectedWorkflows, commit, input, ref); err != nil { log.Error("handle workflows: %v", err) } @@ -289,6 +289,7 @@ func handleWorkflows( detectedWorkflows []*actions_module.DetectedWorkflow, commit *git.Commit, input *notifyInput, + ref string, ) error { if len(detectedWorkflows) == 0 { log.Trace("repo %s with commit %s couldn't find workflows", input.Repo.RepoPath(), commit.ID) @@ -322,7 +323,7 @@ func handleWorkflows( OwnerID: input.Repo.OwnerID, WorkflowID: dwf.EntryName, TriggerUserID: input.Doer.ID, - Ref: input.Ref, + Ref: ref, CommitSHA: commit.ID.String(), IsForkPullRequest: isForkPullRequest, Event: input.Event, @@ -365,11 +366,7 @@ func handleWorkflows( log.Error("FindRunJobs: %v", err) continue } - for _, job := range alljobs { - if err := createCommitStatus(ctx, job); err != nil { - log.Error("CreateCommitStatus: %v", err) - } - } + CreateCommitStatus(ctx, alljobs...) } return nil } From 14d5c46848843e1fb12c16c80678e3506c12dd50 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 23 Aug 2023 10:02:48 +0800 Subject: [PATCH 69/72] Update services/actions/schedule_tasks.go Co-authored-by: wxiaoguang --- services/actions/schedule_tasks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/actions/schedule_tasks.go b/services/actions/schedule_tasks.go index 91d932406c05c..8ed9f771151f6 100644 --- a/services/actions/schedule_tasks.go +++ b/services/actions/schedule_tasks.go @@ -54,7 +54,7 @@ func startTasks(ctx context.Context) error { schedule, err := row.Parse() if err != nil { log.Error("Parse: %v", err) - continue + return err } // Update the spec's next run time and previous run time From 26ce2e9a568a628b9e6c12dbb7fe1ebf11245fd4 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 23 Aug 2023 16:32:06 +0800 Subject: [PATCH 70/72] Fix bug --- services/actions/notifier_helper.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 1d1ba8fdade31..5670defb136ba 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -189,14 +189,10 @@ func notify(ctx context.Context, input *notifyInput) error { } if err := handleSchedules(ctx, schedules, commit, input); err != nil { - log.Error("handle schedules: %v", err) - } - - if err := handleWorkflows(ctx, detectedWorkflows, commit, input, ref); err != nil { - log.Error("handle workflows: %v", err) + return err } - return nil + return handleWorkflows(ctx, detectedWorkflows, commit, input, ref) } func newNotifyInputFromIssue(issue *issues_model.Issue, event webhook_module.HookEventType) *notifyInput { @@ -328,14 +324,15 @@ func handleWorkflows( IsForkPullRequest: isForkPullRequest, Event: input.Event, EventPayload: string(p), + TriggerEvent: dwf.TriggerEvent, Status: actions_model.StatusWaiting, } - need, err := ifNeedApproval(ctx, run, input.Repo, input.Doer) - if err != nil { + if need, err := ifNeedApproval(ctx, run, input.Repo, input.Doer); err != nil { log.Error("check if need approval for repo %d with user %d: %v", input.Repo.ID, input.Doer.ID, err) continue + } else { + run.NeedApproval = need } - run.NeedApproval = need jobs, err := jobparser.Parse(dwf.Content) if err != nil { From 83ec4c624d7586ddf51b0895a655d5c5c39fbbd9 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 23 Aug 2023 16:37:47 +0800 Subject: [PATCH 71/72] make code review easier --- services/actions/notifier_helper.go | 170 ++++++++++++++-------------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 5670defb136ba..ff00e48c644d1 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -195,91 +195,6 @@ func notify(ctx context.Context, input *notifyInput) error { return handleWorkflows(ctx, detectedWorkflows, commit, input, ref) } -func newNotifyInputFromIssue(issue *issues_model.Issue, event webhook_module.HookEventType) *notifyInput { - return newNotifyInput(issue.Repo, issue.Poster, event) -} - -func notifyRelease(ctx context.Context, doer *user_model.User, rel *repo_model.Release, action api.HookReleaseAction) { - if err := rel.LoadAttributes(ctx); err != nil { - log.Error("LoadAttributes: %v", err) - return - } - - permission, _ := access_model.GetUserRepoPermission(ctx, rel.Repo, doer) - - newNotifyInput(rel.Repo, doer, webhook_module.HookEventRelease). - WithRef(git.RefNameFromTag(rel.TagName).String()). - WithPayload(&api.ReleasePayload{ - Action: action, - Release: convert.ToAPIRelease(ctx, rel.Repo, rel), - Repository: convert.ToRepo(ctx, rel.Repo, permission), - Sender: convert.ToUser(ctx, doer, nil), - }). - Notify(ctx) -} - -func notifyPackage(ctx context.Context, sender *user_model.User, pd *packages_model.PackageDescriptor, action api.HookPackageAction) { - if pd.Repository == nil { - // When a package is uploaded to an organization, it could trigger an event to notify. - // So the repository could be nil, however, actions can't support that yet. - // See https://github.com/go-gitea/gitea/pull/17940 - return - } - - apiPackage, err := convert.ToPackage(ctx, pd, sender) - if err != nil { - log.Error("Error converting package: %v", err) - return - } - - newNotifyInput(pd.Repository, sender, webhook_module.HookEventPackage). - WithPayload(&api.PackagePayload{ - Action: action, - Package: apiPackage, - Sender: convert.ToUser(ctx, sender, nil), - }). - Notify(ctx) -} - -func ifNeedApproval(ctx context.Context, run *actions_model.ActionRun, repo *repo_model.Repository, user *user_model.User) (bool, error) { - // 1. don't need approval if it's not a fork PR - // 2. don't need approval if the event is `pull_request_target` since the workflow will run in the context of base branch - // see https://docs.github.com/en/actions/managing-workflow-runs/approving-workflow-runs-from-public-forks#about-workflow-runs-from-public-forks - if !run.IsForkPullRequest || run.TriggerEvent == actions_module.GithubEventPullRequestTarget { - return false, nil - } - - // always need approval if the user is restricted - if user.IsRestricted { - log.Trace("need approval because user %d is restricted", user.ID) - return true, nil - } - - // don't need approval if the user can write - if perm, err := access_model.GetUserRepoPermission(ctx, repo, user); err != nil { - return false, fmt.Errorf("GetUserRepoPermission: %w", err) - } else if perm.CanWrite(unit_model.TypeActions) { - log.Trace("do not need approval because user %d can write", user.ID) - return false, nil - } - - // don't need approval if the user has been approved before - if count, err := actions_model.CountRuns(ctx, actions_model.FindRunOptions{ - RepoID: repo.ID, - TriggerUserID: user.ID, - Approved: true, - }); err != nil { - return false, fmt.Errorf("CountRuns: %w", err) - } else if count > 0 { - log.Trace("do not need approval because user %d has been approved before", user.ID) - return false, nil - } - - // otherwise, need approval - log.Trace("need approval because it's the first time user %d triggered actions", user.ID) - return true, nil -} - func handleWorkflows( ctx context.Context, detectedWorkflows []*actions_module.DetectedWorkflow, @@ -368,6 +283,91 @@ func handleWorkflows( return nil } +func newNotifyInputFromIssue(issue *issues_model.Issue, event webhook_module.HookEventType) *notifyInput { + return newNotifyInput(issue.Repo, issue.Poster, event) +} + +func notifyRelease(ctx context.Context, doer *user_model.User, rel *repo_model.Release, action api.HookReleaseAction) { + if err := rel.LoadAttributes(ctx); err != nil { + log.Error("LoadAttributes: %v", err) + return + } + + permission, _ := access_model.GetUserRepoPermission(ctx, rel.Repo, doer) + + newNotifyInput(rel.Repo, doer, webhook_module.HookEventRelease). + WithRef(git.RefNameFromTag(rel.TagName).String()). + WithPayload(&api.ReleasePayload{ + Action: action, + Release: convert.ToAPIRelease(ctx, rel.Repo, rel), + Repository: convert.ToRepo(ctx, rel.Repo, permission), + Sender: convert.ToUser(ctx, doer, nil), + }). + Notify(ctx) +} + +func notifyPackage(ctx context.Context, sender *user_model.User, pd *packages_model.PackageDescriptor, action api.HookPackageAction) { + if pd.Repository == nil { + // When a package is uploaded to an organization, it could trigger an event to notify. + // So the repository could be nil, however, actions can't support that yet. + // See https://github.com/go-gitea/gitea/pull/17940 + return + } + + apiPackage, err := convert.ToPackage(ctx, pd, sender) + if err != nil { + log.Error("Error converting package: %v", err) + return + } + + newNotifyInput(pd.Repository, sender, webhook_module.HookEventPackage). + WithPayload(&api.PackagePayload{ + Action: action, + Package: apiPackage, + Sender: convert.ToUser(ctx, sender, nil), + }). + Notify(ctx) +} + +func ifNeedApproval(ctx context.Context, run *actions_model.ActionRun, repo *repo_model.Repository, user *user_model.User) (bool, error) { + // 1. don't need approval if it's not a fork PR + // 2. don't need approval if the event is `pull_request_target` since the workflow will run in the context of base branch + // see https://docs.github.com/en/actions/managing-workflow-runs/approving-workflow-runs-from-public-forks#about-workflow-runs-from-public-forks + if !run.IsForkPullRequest || run.TriggerEvent == actions_module.GithubEventPullRequestTarget { + return false, nil + } + + // always need approval if the user is restricted + if user.IsRestricted { + log.Trace("need approval because user %d is restricted", user.ID) + return true, nil + } + + // don't need approval if the user can write + if perm, err := access_model.GetUserRepoPermission(ctx, repo, user); err != nil { + return false, fmt.Errorf("GetUserRepoPermission: %w", err) + } else if perm.CanWrite(unit_model.TypeActions) { + log.Trace("do not need approval because user %d can write", user.ID) + return false, nil + } + + // don't need approval if the user has been approved before + if count, err := actions_model.CountRuns(ctx, actions_model.FindRunOptions{ + RepoID: repo.ID, + TriggerUserID: user.ID, + Approved: true, + }); err != nil { + return false, fmt.Errorf("CountRuns: %w", err) + } else if count > 0 { + log.Trace("do not need approval because user %d has been approved before", user.ID) + return false, nil + } + + // otherwise, need approval + log.Trace("need approval because it's the first time user %d triggered actions", user.ID) + return true, nil +} + func handleSchedules( ctx context.Context, detectedWorkflows []*actions_module.DetectedWorkflow, From 8152ab754f4104ca9416a6a02aff6e622b5d232a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 23 Aug 2023 18:39:23 +0800 Subject: [PATCH 72/72] Fix bug --- models/actions/schedule.go | 6 ++++-- models/actions/schedule_spec.go | 3 +-- services/actions/schedule_tasks.go | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/models/actions/schedule.go b/models/actions/schedule.go index 07a4a0f5ce5d7..b0bc40dadc7dd 100644 --- a/models/actions/schedule.go +++ b/models/actions/schedule.go @@ -52,6 +52,8 @@ func GetReposMapByIDs(ids []int64) (map[int64]*repo_model.Repository, error) { return repos, db.GetEngine(db.DefaultContext).In("id", ids).Find(&repos) } +var cronParser = cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor) + // CreateScheduleTask creates new schedule task. func CreateScheduleTask(ctx context.Context, rows []*ActionSchedule) error { // Return early if there are no rows to insert @@ -75,10 +77,10 @@ func CreateScheduleTask(ctx context.Context, rows []*ActionSchedule) error { // Loop through each schedule spec and create a new spec row now := time.Now() - p := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor) + for _, spec := range row.Specs { // Parse the spec and check for errors - schedule, err := p.Parse(spec) + schedule, err := cronParser.Parse(spec) if err != nil { continue // skip to the next spec if there's an error } diff --git a/models/actions/schedule_spec.go b/models/actions/schedule_spec.go index 6d7876b1ae87a..91240459a044f 100644 --- a/models/actions/schedule_spec.go +++ b/models/actions/schedule_spec.go @@ -33,8 +33,7 @@ type ActionScheduleSpec struct { } func (s *ActionScheduleSpec) Parse() (cron.Schedule, error) { - p := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor) - return p.Parse(s.Spec) + return cronParser.Parse(s.Spec) } func init() { diff --git a/services/actions/schedule_tasks.go b/services/actions/schedule_tasks.go index 8ed9f771151f6..87131e0aab3cf 100644 --- a/services/actions/schedule_tasks.go +++ b/services/actions/schedule_tasks.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" + webhook_module "code.gitea.io/gitea/modules/webhook" "github.com/nektos/act/pkg/jobparser" ) @@ -45,6 +46,19 @@ func startTasks(ctx context.Context) error { // Loop through each spec and create a schedule task for it for _, row := range specs { + // cancel running jobs if the event is push + if row.Schedule.Event == webhook_module.HookEventPush { + // cancel running jobs of the same workflow + if err := actions_model.CancelRunningJobs( + ctx, + row.RepoID, + row.Schedule.Ref, + row.Schedule.WorkflowID, + ); err != nil { + log.Error("CancelRunningJobs: %v", err) + } + } + if err := CreateScheduleTask(ctx, row.Schedule); err != nil { log.Error("CreateScheduleTask: %v", err) return err