Skip to content

Commit a3be913

Browse files
mathstufmathstuf
committed
merge_requests: support in-fork pipelines as well
Fixes: #725
1 parent 71487c3 commit a3be913

File tree

7 files changed

+275
-19
lines changed

7 files changed

+275
-19
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ and this project adheres to [0ver](https://0ver.org) (more or less).
2828
- new configuration parameter: `gitlab.maximum_jobs_queue_size`, controlling the queue buffer size
2929
- new label for pipelines and jobs: `source` to indicate the reason the pipeline started
3030
- new label for pipelines and jobs: `source_project` to indicate pipelines for merge requests on forks
31+
- pipelines in forks hosting merge request pipelines may now be selected using the `include_source_pipelines` option
3132

3233
### Changed
3334

‎docs/configuration_syntax.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,10 @@ project_defaults:
284284
# (optional, default: false)
285285
enabled: false
286286

287+
# Monitor merge request pipelines on source projects as well
288+
# (optional, default: false)
289+
include_source_pipelines: false
290+
287291
# Only keep most 'n' recently updated merge requests
288292
# (optional, default: 0 -- disabled/keep every merge request)
289293
most_recent: 0

‎pkg/config/project.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,9 @@ type ProjectPullRefsMergeRequests struct {
8787
// Monitor pipelines related to project merge requests.
8888
Enabled bool `yaml:"enabled"`
8989

90+
// Monitor all pipelines on a merge request, not just those in the main project
91+
IncludeSourcePipelines bool `default:"false" yaml:"include_source_pipelines"`
92+
9093
// Only keep most 'n' recently updated merge requests.
9194
MostRecent uint `default:"0" yaml:"most_recent"`
9295

‎pkg/controller/pipelines.go‎

Lines changed: 35 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"fmt"
66
"reflect"
7+
"strconv"
78

89
log "github.com/sirupsen/logrus"
910
goGitlab "github.com/xanzy/go-gitlab"
@@ -33,24 +34,45 @@ func (c *Controller) PullRefMetrics(ctx context.Context, ref schemas.Ref) error
3334
"ref-kind": ref.Kind,
3435
}
3536

37+
var (
38+
pipelines []*goGitlab.PipelineInfo
39+
err error
40+
)
41+
3642
// We need a different syntax if the ref is a merge-request
37-
var refName string
38-
if ref.Kind == schemas.RefKindMergeRequest {
39-
refName = fmt.Sprintf("refs/merge-requests/%s/head", ref.Name)
43+
if ref.SourceProject == nil {
44+
var refName string
45+
if ref.Kind == schemas.RefKindMergeRequest {
46+
refName = fmt.Sprintf("refs/merge-requests/%s/head", ref.Name)
47+
} else {
48+
refName = ref.Name
49+
}
50+
51+
pipelines, _, err = c.Gitlab.GetProjectPipelines(ctx, ref.Project.Name, &goGitlab.ListProjectPipelinesOptions{
52+
// We only need the most recent pipeline
53+
ListOptions: goGitlab.ListOptions{
54+
PerPage: 1,
55+
Page: 1,
56+
},
57+
Ref: &refName,
58+
})
59+
if err != nil {
60+
return fmt.Errorf("error fetching project pipelines for %s: %v", ref.Project.Name, err)
61+
}
4062
} else {
41-
refName = ref.Name
42-
}
63+
id, err := strconv.Atoi(ref.Name)
64+
if err != nil {
65+
return fmt.Errorf("error converting merge request refname to an integer %s: %v", ref.Name, err)
66+
}
4367

44-
pipelines, _, err := c.Gitlab.GetProjectPipelines(ctx, ref.Project.Name, &goGitlab.ListProjectPipelinesOptions{
45-
// We only need the most recent pipeline
46-
ListOptions: goGitlab.ListOptions{
68+
pipelines, _, err = c.Gitlab.GetMergeRequestPipelines(ctx, ref.Project.Name, id, &goGitlab.ListOptions{
69+
// We only need the most recent pipeline
4770
PerPage: 1,
4871
Page: 1,
49-
},
50-
Ref: &refName,
51-
})
52-
if err != nil {
53-
return fmt.Errorf("error fetching project pipelines for %s: %v", ref.Project.Name, err)
72+
})
73+
if err != nil {
74+
return fmt.Errorf("error fetching project merge requests pipeline for %s!%s: %v", ref.Project.Name, ref.Name, err)
75+
}
5476
}
5577

5678
if len(pipelines) == 0 {

‎pkg/controller/refs.go‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -59,12 +59,21 @@ func (c *Controller) GetRefs(ctx context.Context, p schemas.Project) (
5959
}
6060

6161
if p.Pull.Refs.MergeRequests.Enabled {
62-
if pulledRefs, err = c.Gitlab.GetRefsFromPipelines(
63-
ctx,
64-
p,
65-
schemas.RefKindMergeRequest,
66-
); err != nil {
67-
return
62+
if p.Pull.Refs.MergeRequests.IncludeSourcePipelines {
63+
if pulledRefs, err = c.Gitlab.GetMergeRequestRefsFromPipelines(
64+
ctx,
65+
p,
66+
); err != nil {
67+
return
68+
}
69+
} else {
70+
if pulledRefs, err = c.Gitlab.GetRefsFromPipelines(
71+
ctx,
72+
p,
73+
schemas.RefKindMergeRequest,
74+
); err != nil {
75+
return
76+
}
6877
}
6978

7079
if err = mergo.Merge(&refs, pulledRefs); err != nil {

‎pkg/gitlab/pipelines.go‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"strings"
99
"time"
1010

11+
retryablehttp "github.com/hashicorp/go-retryablehttp"
1112
log "github.com/sirupsen/logrus"
1213
goGitlab "github.com/xanzy/go-gitlab"
1314
"go.opentelemetry.io/otel"
@@ -86,6 +87,103 @@ func (c *Client) GetProjectPipelines(
8687
return pipelines, resp, nil
8788
}
8889

90+
// ListProjectMergeRequests ..
91+
func (c *Client) ListProjectMergeRequests(
92+
ctx context.Context,
93+
projectName string,
94+
options *goGitlab.ListProjectMergeRequestsOptions,
95+
) (
96+
[]*goGitlab.MergeRequest,
97+
*goGitlab.Response,
98+
error,
99+
) {
100+
ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:ListProjectMergeRequests")
101+
defer span.End()
102+
span.SetAttributes(attribute.String("project_name", projectName))
103+
104+
fields := log.Fields{
105+
"project-name": projectName,
106+
}
107+
108+
if options.Page == 0 {
109+
options.Page = 1
110+
}
111+
112+
if options.PerPage == 0 {
113+
options.PerPage = 100
114+
}
115+
116+
fields["page"] = options.Page
117+
log.WithFields(fields).Trace("listing project merge requests")
118+
119+
c.rateLimit(ctx)
120+
121+
mrs, resp, err := c.MergeRequests.ListProjectMergeRequests(projectName, options, goGitlab.WithContext(ctx))
122+
if err != nil {
123+
return nil, resp, fmt.Errorf("error listing project merge requests for project %s: %s", projectName, err.Error())
124+
}
125+
126+
c.requestsRemaining(resp)
127+
128+
return mrs, resp, nil
129+
}
130+
131+
// GetMergeRequestPipelines ..
132+
func (c *Client) GetMergeRequestPipelines(
133+
ctx context.Context,
134+
projectName string,
135+
mergeRequest int,
136+
options *goGitlab.ListOptions,
137+
) (
138+
[]*goGitlab.PipelineInfo,
139+
*goGitlab.Response,
140+
error,
141+
) {
142+
ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetMergeRequestPipelines")
143+
defer span.End()
144+
span.SetAttributes(attribute.String("project_name", projectName))
145+
146+
fields := log.Fields{
147+
"project-name": projectName,
148+
}
149+
150+
if options.Page == 0 {
151+
options.Page = 1
152+
}
153+
154+
if options.PerPage == 0 {
155+
options.PerPage = 100
156+
}
157+
158+
fields["page"] = options.Page
159+
log.WithFields(fields).Trace("listing merge request pipelines")
160+
161+
params := map[string]string{
162+
"page": fmt.Sprint(options.Page),
163+
"per_page": fmt.Sprint(options.PerPage),
164+
}
165+
optionFunc := func(req *retryablehttp.Request) error {
166+
for k, v := range params {
167+
q := req.URL.Query()
168+
q.Add(k, v)
169+
req.URL.RawQuery = q.Encode()
170+
}
171+
172+
return nil
173+
}
174+
175+
c.rateLimit(ctx)
176+
177+
pipelines, resp, err := c.MergeRequests.ListMergeRequestPipelines(projectName, mergeRequest, optionFunc, goGitlab.WithContext(ctx))
178+
if err != nil {
179+
return nil, resp, fmt.Errorf("error merge request project pipelines for project %s!%d: %s", projectName, mergeRequest, err.Error())
180+
}
181+
182+
c.requestsRemaining(resp)
183+
184+
return pipelines, resp, nil
185+
}
186+
89187
// GetRefPipelineVariablesAsConcatenatedString ..
90188
func (c *Client) GetRefPipelineVariablesAsConcatenatedString(ctx context.Context, ref schemas.Ref) (string, error) {
91189
ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetRefPipelineVariablesAsConcatenatedString")
@@ -292,6 +390,108 @@ func (c *Client) GetRefsFromPipelines(ctx context.Context, p schemas.Project, re
292390
return
293391
}
294392

393+
// GetMergeRequestRefsFromPipelines ..
394+
func (c *Client) GetMergeRequestRefsFromPipelines(ctx context.Context, p schemas.Project) (refs schemas.Refs, err error) {
395+
ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetMergeRequestRefsFromPipelines")
396+
defer span.End()
397+
span.SetAttributes(attribute.String("project_name", p.Name))
398+
span.SetAttributes(attribute.String("ref_kind", "merge-request"))
399+
400+
refs = make(schemas.Refs)
401+
402+
options := &goGitlab.ListProjectMergeRequestsOptions{
403+
ListOptions: goGitlab.ListOptions{
404+
Page: 1,
405+
PerPage: 100,
406+
},
407+
OrderBy: goGitlab.String("updated_at"),
408+
}
409+
410+
var (
411+
mostRecent, maxAgeSeconds uint
412+
limitToMostRecent bool
413+
)
414+
415+
maxAgeSeconds = p.Pull.Refs.MergeRequests.MaxAgeSeconds
416+
mostRecent = p.Pull.Refs.MergeRequests.MostRecent
417+
418+
if mostRecent > 0 {
419+
limitToMostRecent = true
420+
}
421+
422+
if maxAgeSeconds > 0 {
423+
t := time.Now().Add(-time.Second * time.Duration(maxAgeSeconds))
424+
options.UpdatedAfter = &t
425+
}
426+
427+
for {
428+
var (
429+
mrs []*goGitlab.MergeRequest
430+
resp *goGitlab.Response
431+
)
432+
433+
mrs, resp, err = c.ListProjectMergeRequests(ctx, p.Name, options)
434+
if err != nil {
435+
return
436+
}
437+
438+
for _, mr := range mrs {
439+
var refName string
440+
441+
refName = fmt.Sprint(mr.IID)
442+
443+
ref := schemas.NewRef(
444+
p,
445+
schemas.RefKindMergeRequest,
446+
refName,
447+
)
448+
449+
if mr.SourceProjectID == mr.TargetProjectID {
450+
ref.SourceProject = &p
451+
} else {
452+
var (
453+
goProject *goGitlab.Project
454+
sourceProject schemas.Project
455+
)
456+
457+
goProject, err = c.GetProjectByID(ctx, mr.SourceProjectID)
458+
if err != nil {
459+
return
460+
}
461+
sourceProject = p
462+
sourceProject.Name = goProject.PathWithNamespace
463+
464+
ref.SourceProject = &sourceProject
465+
}
466+
467+
if _, ok := refs[ref.Key()]; !ok {
468+
log.WithFields(log.Fields{
469+
"project-name": ref.Project.Name,
470+
"ref": ref.Name,
471+
"ref-kind": ref.Kind,
472+
}).Trace("found ref")
473+
474+
refs[ref.Key()] = ref
475+
476+
if limitToMostRecent {
477+
mostRecent--
478+
if mostRecent <= 0 {
479+
return
480+
}
481+
}
482+
}
483+
}
484+
485+
if resp.CurrentPage >= resp.NextPage {
486+
break
487+
}
488+
489+
options.Page = resp.NextPage
490+
}
491+
492+
return
493+
}
494+
295495
// GetRefPipelineTestReport ..
296496
func (c *Client) GetRefPipelineTestReport(ctx context.Context, ref schemas.Ref) (schemas.TestReport, error) {
297497
ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetRefPipelineTestReport")

‎pkg/gitlab/projects.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,23 @@ func (c *Client) GetProject(ctx context.Context, name string) (*goGitlab.Project
3232
return p, err
3333
}
3434

35+
// GetProjectByID ..
36+
func (c *Client) GetProjectByID(ctx context.Context, id int) (*goGitlab.Project, error) {
37+
ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetProjectByID")
38+
defer span.End()
39+
span.SetAttributes(attribute.Int("project_id", id))
40+
41+
log.WithFields(log.Fields{
42+
"project-id": id,
43+
}).Debug("reading project")
44+
45+
c.rateLimit(ctx)
46+
p, resp, err := c.Projects.GetProject(id, &goGitlab.GetProjectOptions{}, goGitlab.WithContext(ctx))
47+
c.requestsRemaining(resp)
48+
49+
return p, err
50+
}
51+
3552
// ListProjects ..
3653
func (c *Client) ListProjects(ctx context.Context, w config.Wildcard) ([]schemas.Project, error) {
3754
ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:ListProjects")

0 commit comments

Comments
 (0)