From 8cfa8492be48f0ea7e66e949ea1393d6518bff66 Mon Sep 17 00:00:00 2001 From: Janilton Maciel Date: Thu, 13 May 2021 17:20:44 -0300 Subject: [PATCH 1/2] process rules in parallel --- collector/collector.go | 34 ++++++++++++++++++++-------------- rules/registry.go | 11 ++++++++++- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/collector/collector.go b/collector/collector.go index 9f9e6fb..11a2b34 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -4,6 +4,7 @@ package main import ( + "sync" "time" log "github.com/sirupsen/logrus" @@ -117,6 +118,20 @@ func insertStats(r *rules.Registry, projectsCount int) error { return nil } +func worker(projects []*gitlab.Project, git *gitlab.Client, wg *sync.WaitGroup) { + defer wg.Done() + for _, project := range projects { + // Only non-forks + if project.ForkedFromProject != nil { + continue + } + + for _, rulesFn := range rules.MyRegistry.RulesFn { + rules.MyRegistry.ProcessProject(git, project, rulesFn) + } + } +} + func main() { git, err := gitlab.NewClient( viper.GetString("gitlab.token"), @@ -140,36 +155,27 @@ func main() { } var gitlabProjectsCount int + var wg sync.WaitGroup for { projects, resp, err := git.Projects.ListProjects(opt) if err != nil { log.Fatal(err) } - - gitlabProjectsCount = resp.TotalItems - log.Debugf( "[Collector] Page %d of %d", resp.CurrentPage, resp.TotalPages, ) - for _, project := range projects { - // Only non-forks - if project.ForkedFromProject != nil { - continue - } - - for _, rulesFn := range rules.MyRegistry.RulesFn { - rules.MyRegistry.ProcessProject(git, project, rulesFn) - } - } + wg.Add(1) + go worker(projects, git, &wg) + gitlabProjectsCount = resp.TotalItems if resp.CurrentPage >= resp.TotalPages { break } - opt.Page = resp.NextPage } + wg.Wait() if err := processRules(rules.MyRegistry.Rules); err != nil { log.Errorf("[Collector] Error on insert rules data: %v", err) diff --git a/rules/registry.go b/rules/registry.go index 947b8c3..e23174d 100644 --- a/rules/registry.go +++ b/rules/registry.go @@ -3,9 +3,14 @@ package rules -import "github.com/xanzy/go-gitlab" +import ( + "sync" + + "github.com/xanzy/go-gitlab" +) type Registry struct { + mu sync.Mutex Projects map[string]Project Rules []Rule RulesFn map[string]Ruler @@ -23,6 +28,10 @@ func (r *Registry) ProcessProject(c *gitlab.Client, p *gitlab.Project, ruler Rul if !result { return false } + + r.mu.Lock() + defer r.mu.Unlock() + rule := NewRule(p, ruler) r.Rules = append(r.Rules, rule) From 99abfde90a2e7bf54e2fa184f60b62d4329672e6 Mon Sep 17 00:00:00 2001 From: Janilton Maciel Date: Fri, 14 May 2021 15:14:50 -0300 Subject: [PATCH 2/2] refactor rules --- rules/empty_repository.go | 35 -------------------------- rules/fast-forward-merge.go | 34 -------------------------- rules/has_open_issues.go | 37 ---------------------------- rules/last_activity.go | 41 ------------------------------- rules/my_registry.go | 19 --------------- rules/registry.go | 12 ++++++--- rules/rule/empty_repository.go | 25 +++++++++++++++++++ rules/rule/fast-forward-merge.go | 25 +++++++++++++++++++ rules/rule/has_open_issues.go | 25 +++++++++++++++++++ rules/rule/last_activity.go | 29 ++++++++++++++++++++++ rules/rule/rule.go | 17 +++++++++++++ rules/rule/without_gitlab_ci.go | 32 ++++++++++++++++++++++++ rules/rule/without_readme.go | 25 +++++++++++++++++++ rules/ruler.go | 15 ++++++++++++ rules/without_gitlab_ci.go | 42 -------------------------------- rules/without_readme.go | 34 -------------------------- 16 files changed, 202 insertions(+), 245 deletions(-) delete mode 100644 rules/empty_repository.go delete mode 100644 rules/fast-forward-merge.go delete mode 100644 rules/has_open_issues.go delete mode 100644 rules/last_activity.go delete mode 100644 rules/my_registry.go create mode 100644 rules/rule/empty_repository.go create mode 100644 rules/rule/fast-forward-merge.go create mode 100644 rules/rule/has_open_issues.go create mode 100644 rules/rule/last_activity.go create mode 100644 rules/rule/rule.go create mode 100644 rules/rule/without_gitlab_ci.go create mode 100644 rules/rule/without_readme.go delete mode 100644 rules/without_gitlab_ci.go delete mode 100644 rules/without_readme.go diff --git a/rules/empty_repository.go b/rules/empty_repository.go deleted file mode 100644 index 0537716..0000000 --- a/rules/empty_repository.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) 2021, Marcelo Jorge Vieira -// Licensed under the BSD 3-Clause License - -package rules - -import "github.com/xanzy/go-gitlab" - -type EmptyRepository struct { - Description string `json:"description"` - ID string `json:"ruleId"` - Level string `json:"level"` - Name string `json:"name"` -} - -func (e *EmptyRepository) Run(c *gitlab.Client, p *gitlab.Project) bool { - return p.EmptyRepo -} - -func (e *EmptyRepository) GetSlug() string { - return "empty-repository" -} - -func (e *EmptyRepository) GetLevel() string { - return LevelError -} - -func NewEmptyRepository() Ruler { - e := &EmptyRepository{ - Name: "Empty Repository", - Description: "", - } - e.ID = e.GetSlug() - e.Level = e.GetLevel() - return e -} diff --git a/rules/fast-forward-merge.go b/rules/fast-forward-merge.go deleted file mode 100644 index d9eb816..0000000 --- a/rules/fast-forward-merge.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) 2021, Marcelo Jorge Vieira -// Licensed under the BSD 3-Clause License - -package rules - -import "github.com/xanzy/go-gitlab" - -type NonFastForwardMerge struct { - Description string `json:"description"` - ID string `json:"ruleId"` - Level string `json:"level"` - Name string `json:"name"` -} - -func (w *NonFastForwardMerge) Run(c *gitlab.Client, p *gitlab.Project) bool { - return p.MergeMethod != gitlab.FastForwardMerge -} -func (w *NonFastForwardMerge) GetSlug() string { - return "non-fast-forward-merge" -} - -func (w *NonFastForwardMerge) GetLevel() string { - return LevelPedantic -} - -func NewNonFastForwardMerge() Ruler { - w := &NonFastForwardMerge{ - Name: "Non Fast-forward Merge", - Description: "", - } - w.ID = w.GetSlug() - w.Level = w.GetLevel() - return w -} diff --git a/rules/has_open_issues.go b/rules/has_open_issues.go deleted file mode 100644 index 8f8742b..0000000 --- a/rules/has_open_issues.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2021, Marcelo Jorge Vieira -// Licensed under the BSD 3-Clause License - -package rules - -import ( - "github.com/xanzy/go-gitlab" -) - -type HasOpenIssues struct { - Description string `json:"description"` - ID string `json:"ruleId"` - Level string `json:"level"` - Name string `json:"name"` -} - -func (h *HasOpenIssues) Run(c *gitlab.Client, p *gitlab.Project) bool { - return p.OpenIssuesCount > 0 -} - -func (h *HasOpenIssues) GetSlug() string { - return "has-open-issues" -} - -func (h *HasOpenIssues) GetLevel() string { - return LevelPedantic -} - -func NewHasOpenIssues() Ruler { - h := &HasOpenIssues{ - Name: "Has Open Issues", - Description: "", - } - h.ID = h.GetSlug() - h.Level = h.GetLevel() - return h -} diff --git a/rules/last_activity.go b/rules/last_activity.go deleted file mode 100644 index fefbae2..0000000 --- a/rules/last_activity.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) 2021, Marcelo Jorge Vieira -// Licensed under the BSD 3-Clause License - -package rules - -import ( - "time" - - "github.com/xanzy/go-gitlab" -) - -type LastActivity struct { - Description string `json:"description"` - ID string `json:"ruleId"` - Level string `json:"level"` - Name string `json:"name"` -} - -func (l *LastActivity) Run(c *gitlab.Client, p *gitlab.Project) bool { - t2 := time.Now() - days := t2.Sub(*p.LastActivityAt).Hours() / 24 - return days > 365 -} - -func (l *LastActivity) GetSlug() string { - return "last-activity-1-year" -} - -func (l *LastActivity) GetLevel() string { - return LevelWarning -} - -func NewLastActivity() Ruler { - l := &LastActivity{ - Name: "Last Activity > 1 year", - Description: "", - } - l.ID = l.GetSlug() - l.Level = l.GetLevel() - return l -} diff --git a/rules/my_registry.go b/rules/my_registry.go deleted file mode 100644 index a239002..0000000 --- a/rules/my_registry.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) 2021, Marcelo Jorge Vieira -// Licensed under the BSD 3-Clause License - -package rules - -var MyRegistry = &Registry{ - Projects: map[string]Project{}, - Rules: []Rule{}, - RulesFn: map[string]Ruler{}, -} - -func init() { - MyRegistry.AddRule(NewEmptyRepository()) - MyRegistry.AddRule(NewHasOpenIssues()) - MyRegistry.AddRule(NewLastActivity()) - MyRegistry.AddRule(NewNonFastForwardMerge()) - MyRegistry.AddRule(NewWithoutGitlabCI()) - MyRegistry.AddRule(NewWithoutReadme()) -} diff --git a/rules/registry.go b/rules/registry.go index e23174d..5b98f2a 100644 --- a/rules/registry.go +++ b/rules/registry.go @@ -9,6 +9,12 @@ import ( "github.com/xanzy/go-gitlab" ) +var MyRegistry = &Registry{ + Projects: map[string]Project{}, + Rules: []Rule{}, + RulesFn: map[string]Ruler{}, +} + type Registry struct { mu sync.Mutex Projects map[string]Project @@ -16,11 +22,11 @@ type Registry struct { RulesFn map[string]Ruler } -func (r *Registry) AddRule(ruler Ruler) { - if _, ok := r.RulesFn[ruler.GetSlug()]; ok { +func (r *Registry) AddRule(rule Ruler) { + if _, ok := r.RulesFn[rule.GetSlug()]; ok { return } - r.RulesFn[ruler.GetSlug()] = ruler + r.RulesFn[rule.GetSlug()] = rule } func (r *Registry) ProcessProject(c *gitlab.Client, p *gitlab.Project, ruler Ruler) bool { diff --git a/rules/rule/empty_repository.go b/rules/rule/empty_repository.go new file mode 100644 index 0000000..a88d94e --- /dev/null +++ b/rules/rule/empty_repository.go @@ -0,0 +1,25 @@ +// Copyright (c) 2021, Marcelo Jorge Vieira +// Licensed under the BSD 3-Clause License + +package ruler + +import ( + "github.com/globocom/gitlab-lint/rules" + "github.com/xanzy/go-gitlab" +) + +type EmptyRepository struct { + rules.RulerImpl +} + +func (e *EmptyRepository) Run(c *gitlab.Client, p *gitlab.Project) bool { + return p.EmptyRepo +} + +func NewEmptyRepository() rules.Ruler { + e := new(EmptyRepository) + e.ID = "empty-repository" + e.Name = "Empty Repository" + e.Level = rules.LevelError + return e +} diff --git a/rules/rule/fast-forward-merge.go b/rules/rule/fast-forward-merge.go new file mode 100644 index 0000000..fe2f207 --- /dev/null +++ b/rules/rule/fast-forward-merge.go @@ -0,0 +1,25 @@ +// Copyright (c) 2021, Marcelo Jorge Vieira +// Licensed under the BSD 3-Clause License + +package ruler + +import ( + "github.com/globocom/gitlab-lint/rules" + "github.com/xanzy/go-gitlab" +) + +type NonFastForwardMerge struct { + rules.RulerImpl +} + +func (w *NonFastForwardMerge) Run(c *gitlab.Client, p *gitlab.Project) bool { + return p.MergeMethod != gitlab.FastForwardMerge +} + +func NewNonFastForwardMerge() rules.Ruler { + w := new(NonFastForwardMerge) + w.ID = "non-fast-forward-merge" + w.Name = "Non Fast-forward Merge" + w.Level = rules.LevelPedantic + return w +} diff --git a/rules/rule/has_open_issues.go b/rules/rule/has_open_issues.go new file mode 100644 index 0000000..58a5262 --- /dev/null +++ b/rules/rule/has_open_issues.go @@ -0,0 +1,25 @@ +// Copyright (c) 2021, Marcelo Jorge Vieira +// Licensed under the BSD 3-Clause License + +package ruler + +import ( + "github.com/globocom/gitlab-lint/rules" + "github.com/xanzy/go-gitlab" +) + +type HasOpenIssues struct { + rules.RulerImpl +} + +func (h *HasOpenIssues) Run(c *gitlab.Client, p *gitlab.Project) bool { + return p.OpenIssuesCount > 0 +} + +func NewHasOpenIssues() rules.Ruler { + h := new(HasOpenIssues) + h.ID = "has-open-issues" + h.Name = "Has Open Issues" + h.Level = rules.LevelPedantic + return h +} diff --git a/rules/rule/last_activity.go b/rules/rule/last_activity.go new file mode 100644 index 0000000..80dc3f6 --- /dev/null +++ b/rules/rule/last_activity.go @@ -0,0 +1,29 @@ +// Copyright (c) 2021, Marcelo Jorge Vieira +// Licensed under the BSD 3-Clause License + +package ruler + +import ( + "time" + + "github.com/globocom/gitlab-lint/rules" + "github.com/xanzy/go-gitlab" +) + +type LastActivity struct { + rules.RulerImpl +} + +func (l *LastActivity) Run(c *gitlab.Client, p *gitlab.Project) bool { + t2 := time.Now() + days := t2.Sub(*p.LastActivityAt).Hours() / 24 + return days > 365 +} + +func NewLastActivity() rules.Ruler { + l := new(LastActivity) + l.ID = "last-activity-1-year" + l.Name = "Last Activity > 1 year" + l.Level = rules.LevelWarning + return l +} diff --git a/rules/rule/rule.go b/rules/rule/rule.go new file mode 100644 index 0000000..5af8e7a --- /dev/null +++ b/rules/rule/rule.go @@ -0,0 +1,17 @@ +// Copyright (c) 2021, Marcelo Jorge Vieira +// Licensed under the BSD 3-Clause License + +package ruler + +import ( + "github.com/globocom/gitlab-lint/rules" +) + +func init() { + rules.MyRegistry.AddRule(NewEmptyRepository()) + rules.MyRegistry.AddRule(NewHasOpenIssues()) + rules.MyRegistry.AddRule(NewLastActivity()) + rules.MyRegistry.AddRule(NewNonFastForwardMerge()) + rules.MyRegistry.AddRule(NewWithoutGitlabCI()) + rules.MyRegistry.AddRule(NewWithoutReadme()) +} diff --git a/rules/rule/without_gitlab_ci.go b/rules/rule/without_gitlab_ci.go new file mode 100644 index 0000000..52a9efd --- /dev/null +++ b/rules/rule/without_gitlab_ci.go @@ -0,0 +1,32 @@ +// Copyright (c) 2021, Marcelo Jorge Vieira +// Licensed under the BSD 3-Clause License + +package ruler + +import ( + "github.com/globocom/gitlab-lint/rules" + "github.com/xanzy/go-gitlab" +) + +type WithoutGitlabCI struct { + rules.RulerImpl +} + +func (w *WithoutGitlabCI) Run(c *gitlab.Client, p *gitlab.Project) bool { + gf := &gitlab.GetFileOptions{ + Ref: gitlab.String(p.DefaultBranch), + } + _, _, err := c.RepositoryFiles.GetFile( + p.PathWithNamespace, ".gitlab-ci.yml", gf, + ) + // 404 + return err != nil +} + +func NewWithoutGitlabCI() rules.Ruler { + w := new(WithoutGitlabCI) + w.ID = "without-gitlab-ci" + w.Name = "Without Gitlab CI" + w.Level = rules.LevelInfo + return w +} diff --git a/rules/rule/without_readme.go b/rules/rule/without_readme.go new file mode 100644 index 0000000..9d7492e --- /dev/null +++ b/rules/rule/without_readme.go @@ -0,0 +1,25 @@ +// Copyright (c) 2021, Marcelo Jorge Vieira +// Licensed under the BSD 3-Clause License + +package ruler + +import ( + "github.com/globocom/gitlab-lint/rules" + "github.com/xanzy/go-gitlab" +) + +type WithoutReadme struct { + rules.RulerImpl +} + +func (w *WithoutReadme) Run(c *gitlab.Client, p *gitlab.Project) bool { + return p.ReadmeURL == "" +} + +func NewWithoutReadme() rules.Ruler { + w := new(WithoutReadme) + w.ID = "without-readme" + w.Name = "Without Readme" + w.Level = rules.LevelError + return w +} diff --git a/rules/ruler.go b/rules/ruler.go index 8cdd5f1..9fd5eaa 100644 --- a/rules/ruler.go +++ b/rules/ruler.go @@ -10,3 +10,18 @@ type Ruler interface { GetSlug() string GetLevel() string } + +type RulerImpl struct { + ID string `json:"ruleId"` + Name string `json:"name"` + Description string `json:"description"` + Level string `json:"level"` +} + +func (e RulerImpl) GetSlug() string { + return e.ID +} + +func (e RulerImpl) GetLevel() string { + return e.Level +} diff --git a/rules/without_gitlab_ci.go b/rules/without_gitlab_ci.go deleted file mode 100644 index db0ffd2..0000000 --- a/rules/without_gitlab_ci.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2021, Marcelo Jorge Vieira -// Licensed under the BSD 3-Clause License - -package rules - -import "github.com/xanzy/go-gitlab" - -type WithoutGitlabCI struct { - Description string `json:"description"` - ID string `json:"ruleId"` - Level string `json:"level"` - Name string `json:"name"` -} - -func (w *WithoutGitlabCI) Run(c *gitlab.Client, p *gitlab.Project) bool { - gf := &gitlab.GetFileOptions{ - Ref: gitlab.String(p.DefaultBranch), - } - _, _, err := c.RepositoryFiles.GetFile( - p.PathWithNamespace, ".gitlab-ci.yml", gf, - ) - // 404 - return err != nil -} - -func (w *WithoutGitlabCI) GetSlug() string { - return "without-gitlab-ci" -} - -func (w *WithoutGitlabCI) GetLevel() string { - return LevelInfo -} - -func NewWithoutGitlabCI() Ruler { - w := &WithoutGitlabCI{ - Name: "Without Gitlab CI", - Description: "", - } - w.ID = w.GetSlug() - w.Level = w.GetLevel() - return w -} diff --git a/rules/without_readme.go b/rules/without_readme.go deleted file mode 100644 index 90096b9..0000000 --- a/rules/without_readme.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) 2021, Marcelo Jorge Vieira -// Licensed under the BSD 3-Clause License - -package rules - -import "github.com/xanzy/go-gitlab" - -type WithoutReadme struct { - Description string `json:"description"` - ID string `json:"ruleId"` - Level string `json:"level"` - Name string `json:"name"` -} - -func (w *WithoutReadme) Run(c *gitlab.Client, p *gitlab.Project) bool { - return p.ReadmeURL == "" -} -func (w *WithoutReadme) GetSlug() string { - return "without-readme" -} - -func (w *WithoutReadme) GetLevel() string { - return LevelError -} - -func NewWithoutReadme() Ruler { - w := &WithoutReadme{ - Name: "Without Readme", - Description: "", - } - w.ID = w.GetSlug() - w.Level = w.GetLevel() - return w -}