Skip to content

Commit 78af0e1

Browse files
calmezcalmez
authored
feat: enabled multiple metrics per ref (#934)
* extended pipelines being fetched per ref this allows multiple metrics on one ref to be published * added backwards compatible config for fetching multiple pipelines per ref * s/Mutext/Mutex * made api use backwards compatible to vanilla code * comment for hard to understand code * added test to show the bug * implemented pipeline caching to do proper comparisons for updates
1 parent 461062d commit 78af0e1

File tree

17 files changed

+375
-39
lines changed

17 files changed

+375
-39
lines changed

‎pkg/config/config_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ func TestNew(t *testing.T) {
7474
c.ProjectDefaults.Pull.Pipeline.Jobs.RunnerDescription.Enabled = true
7575
c.ProjectDefaults.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp = `shared-runners-manager-(\d*)\.gitlab\.com`
7676
c.ProjectDefaults.Pull.Pipeline.Variables.Regexp = `.*`
77+
c.ProjectDefaults.Pull.Pipeline.PerRef = 1
7778

7879
c.Redis.ProjectTTL = 168 * time.Hour
7980
c.Redis.RefTTL = 1 * time.Hour

‎pkg/config/parser_test.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ project_defaults:
116116
variables:
117117
enabled: true
118118
regexp: "^CI_"
119+
per_ref: 3
119120
120121
projects:
121122
- name: foo/project
@@ -128,6 +129,8 @@ projects:
128129
branches:
129130
regexp: "^foo$"
130131
max_age_seconds: 2
132+
pipeline:
133+
per_ref: 10
131134
132135
wildcards:
133136
- owner:
@@ -228,6 +231,7 @@ wildcards:
228231
xcfg.ProjectDefaults.Pull.Pipeline.Jobs.Enabled = true
229232
xcfg.ProjectDefaults.Pull.Pipeline.Variables.Enabled = true
230233
xcfg.ProjectDefaults.Pull.Pipeline.Variables.Regexp = `^CI_`
234+
xcfg.ProjectDefaults.Pull.Pipeline.PerRef = 3
231235

232236
p1 := NewProject("foo/project")
233237
p1.ProjectParameters = xcfg.ProjectDefaults
@@ -239,6 +243,7 @@ wildcards:
239243
p2.Pull.Environments.Regexp = `^foo$`
240244
p2.Pull.Refs.Branches.Regexp = `^foo$`
241245
p2.Pull.Refs.Branches.MaxAgeSeconds = 2
246+
p2.Pull.Pipeline.PerRef = 10
242247

243248
xcfg.Projects = []Project{p1, p2}
244249

‎pkg/config/project.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ type ProjectPullPipeline struct {
106106
Jobs ProjectPullPipelineJobs `yaml:"jobs"`
107107
Variables ProjectPullPipelineVariables `yaml:"variables"`
108108
TestReports ProjectPullPipelineTestReports `yaml:"test_reports"`
109+
PerRef uint `default:"1" yaml:"per_ref"`
109110
}
110111

111112
// ProjectPullPipelineJobs ..

‎pkg/config/project_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ func TestNewProject(t *testing.T) {
3030
p.Pull.Pipeline.Jobs.RunnerDescription.Enabled = true
3131
p.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp = `shared-runners-manager-(\d*)\.gitlab\.com`
3232
p.Pull.Pipeline.Variables.Regexp = `.*`
33+
p.Pull.Pipeline.PerRef = 1
3334

3435
assert.Equal(t, p, NewProject("foo/bar"))
3536
}

‎pkg/config/wildcard_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ func TestNewWildcard(t *testing.T) {
2828
w.Pull.Pipeline.Jobs.RunnerDescription.Enabled = true
2929
w.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp = `shared-runners-manager-(\d*)\.gitlab\.com`
3030
w.Pull.Pipeline.Variables.Regexp = `.*`
31+
w.Pull.Pipeline.PerRef = 1
3132

3233
assert.Equal(t, w, NewWildcard())
3334
}

‎pkg/controller/pipelines.go‎

Lines changed: 47 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,6 @@ import (
1414

1515
// PullRefMetrics ..
1616
func (c *Controller) PullRefMetrics(ctx context.Context, ref schemas.Ref) error {
17-
finishedStatusesList := []string{
18-
"success",
19-
"failed",
20-
"skipped",
21-
"cancelled",
22-
}
23-
2417
// At scale, the scheduled ref may be behind the actual state being stored
2518
// to avoid issues, we refresh it from the store before manipulating it
2619
if err := c.Store.GetRef(ctx, &ref); err != nil {
@@ -42,9 +35,8 @@ func (c *Controller) PullRefMetrics(ctx context.Context, ref schemas.Ref) error
4235
}
4336

4437
pipelines, _, err := c.Gitlab.GetProjectPipelines(ctx, ref.Project.Name, &goGitlab.ListProjectPipelinesOptions{
45-
// We only need the most recent pipeline
4638
ListOptions: goGitlab.ListOptions{
47-
PerPage: 1,
39+
PerPage: int(ref.Project.Pull.Pipeline.PerRef),
4840
Page: 1,
4941
},
5042
Ref: &refName,
@@ -74,21 +66,59 @@ func (c *Controller) PullRefMetrics(ctx context.Context, ref schemas.Ref) error
7466
return nil
7567
}
7668

77-
pipeline, err := c.Gitlab.GetRefPipeline(ctx, ref, pipelines[0].ID)
69+
// Reverse result list to have `ref`'s `LatestPipeline` untouched (compared to
70+
// default behavior) after looping over list
71+
slices.Reverse(pipelines)
72+
73+
for _, apiPipeline := range pipelines {
74+
err := c.ProcessPipelinesMetrics(ctx, ref, apiPipeline)
75+
if err != nil {
76+
log.WithFields(log.Fields{
77+
"pipeline": apiPipeline.ID,
78+
"error": err,
79+
}).Error("processing pipeline metrics failed")
80+
}
81+
}
82+
83+
return nil
84+
}
85+
86+
func (c *Controller) ProcessPipelinesMetrics(ctx context.Context, ref schemas.Ref, apiPipeline *goGitlab.PipelineInfo) error {
87+
finishedStatusesList := []string{
88+
"success",
89+
"failed",
90+
"skipped",
91+
"cancelled",
92+
}
93+
94+
pipeline, err := c.Gitlab.GetRefPipeline(ctx, ref, apiPipeline.ID)
7895
if err != nil {
7996
return err
8097
}
8198

82-
if ref.LatestPipeline.ID == 0 || !reflect.DeepEqual(pipeline, ref.LatestPipeline) {
83-
formerPipeline := ref.LatestPipeline
84-
ref.LatestPipeline = pipeline
85-
86-
// fetch pipeline variables
87-
if ref.Project.Pull.Pipeline.Variables.Enabled {
88-
ref.LatestPipeline.Variables, err = c.Gitlab.GetRefPipelineVariablesAsConcatenatedString(ctx, ref)
99+
// fetch pipeline variables
100+
if ref.Project.Pull.Pipeline.Variables.Enabled {
101+
if exists, _ := c.Store.PipelineVariablesExists(ctx, pipeline); !exists {
102+
variables, err := c.Gitlab.GetRefPipelineVariablesAsConcatenatedString(ctx, ref, pipeline)
103+
c.Store.SetPipelineVariables(ctx, pipeline, variables)
104+
pipeline.Variables = variables
89105
if err != nil {
90106
return err
91107
}
108+
} else {
109+
variables, _ := c.Store.GetPipelineVariables(ctx, pipeline)
110+
pipeline.Variables = variables
111+
}
112+
}
113+
114+
var cachedPipeline schemas.Pipeline
115+
116+
if c.Store.GetPipeline(ctx, &cachedPipeline); cachedPipeline.ID == 0 || !reflect.DeepEqual(pipeline, cachedPipeline) {
117+
formerPipeline := ref.LatestPipeline
118+
ref.LatestPipeline = pipeline
119+
120+
if err = c.Store.SetPipeline(ctx, pipeline); err != nil {
121+
return err
92122
}
93123

94124
// Update the ref in the store

‎pkg/controller/pipelines_test.go‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,127 @@ func TestPullRefMetricsSucceed(t *testing.T) {
101101
assert.Equal(t, status, metrics[status.Key()])
102102
}
103103

104+
func TestPullRefMetricsUpdatingPipeline(t *testing.T) {
105+
// given
106+
ctx, c, mux, srv := newTestController(config.Config{})
107+
defer srv.Close()
108+
apiPipeline := `{
109+
"id":1,
110+
"created_at":"2016-08-11T11:27:00.085Z",
111+
"started_at":"2016-08-11T11:28:00.085Z",
112+
"duration":300,
113+
"queued_duration":60,
114+
"status":"running",
115+
"coverage":"30.2",
116+
"source":"pipeline"
117+
}`
118+
119+
mux.HandleFunc("/api/v4/projects/foo/pipelines",
120+
func(w http.ResponseWriter, r *http.Request) {
121+
assert.Equal(t, "bar", r.URL.Query().Get("ref"))
122+
fmt.Fprint(w, `[{"id":1}]`)
123+
})
124+
125+
mux.HandleFunc("/api/v4/projects/foo/pipelines/1",
126+
func(w http.ResponseWriter, r *http.Request) {
127+
fmt.Fprint(w, apiPipeline)
128+
})
129+
130+
mux.HandleFunc("/api/v4/projects/foo/pipelines/1/variables",
131+
func(w http.ResponseWriter, r *http.Request) {
132+
assert.Equal(t, "GET", r.Method)
133+
fmt.Fprint(w, `[{"key":"foo","value":"bar"}]`)
134+
})
135+
136+
p := schemas.NewProject("foo")
137+
p.Pull.Pipeline.Variables.Enabled = true
138+
139+
labels := map[string]string{
140+
"kind": string(schemas.RefKindBranch),
141+
"project": "foo",
142+
"ref": "bar",
143+
"topics": "",
144+
"variables": "foo:bar",
145+
"source": "pipeline",
146+
}
147+
148+
// when
149+
assert.NoError(t, c.PullRefMetrics(
150+
ctx,
151+
schemas.NewRef(
152+
p,
153+
schemas.RefKindBranch,
154+
"bar",
155+
)))
156+
157+
metrics, _ := c.Store.Metrics(ctx)
158+
159+
// then
160+
runID := schemas.Metric{
161+
Kind: schemas.MetricKindID,
162+
Labels: labels,
163+
Value: 1,
164+
}
165+
assert.Equal(t, runID, metrics[runID.Key()])
166+
167+
labels["status"] = "running"
168+
status := schemas.Metric{
169+
Kind: schemas.MetricKindStatus,
170+
Labels: labels,
171+
Value: 1,
172+
}
173+
assert.Equal(t, status, metrics[status.Key()])
174+
175+
// given again
176+
apiPipeline = `{
177+
"id":1,
178+
"created_at":"2016-08-11T11:27:00.085Z",
179+
"started_at":"2016-08-11T11:28:00.085Z",
180+
"duration":300,
181+
"queued_duration":60,
182+
"status":"failed",
183+
"coverage":"30.2",
184+
"source":"pipeline"
185+
}`
186+
187+
labels = map[string]string{
188+
"kind": string(schemas.RefKindBranch),
189+
"project": "foo",
190+
"ref": "bar",
191+
"topics": "",
192+
"variables": "foo:bar",
193+
"source": "pipeline",
194+
}
195+
196+
// when again
197+
assert.NoError(t, c.PullRefMetrics(
198+
ctx,
199+
schemas.NewRef(
200+
p,
201+
schemas.RefKindBranch,
202+
"bar",
203+
)))
204+
205+
metrics, _ = c.Store.Metrics(ctx)
206+
207+
// then again
208+
runID = schemas.Metric{
209+
Kind: schemas.MetricKindID,
210+
Labels: labels,
211+
Value: 1,
212+
}
213+
assert.Equal(t, runID, metrics[runID.Key()])
214+
215+
labels["status"] = "failed"
216+
status = schemas.Metric{
217+
Kind: schemas.MetricKindStatus,
218+
Labels: labels,
219+
Value: 1,
220+
}
221+
assert.Equal(t, status, metrics[status.Key()])
222+
223+
}
224+
104225
func TestPullRefTestReportMetrics(t *testing.T) {
105226
ctx, c, mux, srv := newTestController(config.Config{})
106227
defer srv.Close()

‎pkg/gitlab/pipelines.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -88,19 +88,19 @@ func (c *Client) GetProjectPipelines(
8888
}
8989

9090
// GetRefPipelineVariablesAsConcatenatedString ..
91-
func (c *Client) GetRefPipelineVariablesAsConcatenatedString(ctx context.Context, ref schemas.Ref) (string, error) {
91+
func (c *Client) GetRefPipelineVariablesAsConcatenatedString(ctx context.Context, ref schemas.Ref, pipeline schemas.Pipeline) (string, error) {
9292
ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetRefPipelineVariablesAsConcatenatedString")
9393
defer span.End()
9494
span.SetAttributes(attribute.String("project_name", ref.Project.Name))
9595
span.SetAttributes(attribute.String("ref_name", ref.Name))
9696

97-
if reflect.DeepEqual(ref.LatestPipeline, (schemas.Pipeline{})) {
97+
if reflect.DeepEqual(pipeline, (schemas.Pipeline{})) {
9898
log.WithFields(
9999
log.Fields{
100100
"project-name": ref.Project.Name,
101101
"ref": ref.Name,
102102
},
103-
).Debug("most recent pipeline not defined, exiting..")
103+
).Debug("pipeline not defined, exiting..")
104104

105105
return "", nil
106106
}
@@ -109,7 +109,7 @@ func (c *Client) GetRefPipelineVariablesAsConcatenatedString(ctx context.Context
109109
log.Fields{
110110
"project-name": ref.Project.Name,
111111
"ref": ref.Name,
112-
"pipeline-id": ref.LatestPipeline.ID,
112+
"pipeline-id": pipeline.ID,
113113
},
114114
).Debug("fetching pipeline variables")
115115

@@ -124,9 +124,9 @@ func (c *Client) GetRefPipelineVariablesAsConcatenatedString(ctx context.Context
124124

125125
c.rateLimit(ctx)
126126

127-
variables, resp, err := c.Pipelines.GetPipelineVariables(ref.Project.Name, ref.LatestPipeline.ID, goGitlab.WithContext(ctx))
127+
variables, resp, err := c.Pipelines.GetPipelineVariables(ref.Project.Name, pipeline.ID, goGitlab.WithContext(ctx))
128128
if err != nil {
129-
return "", fmt.Errorf("could not fetch pipeline variables for %d: %s", ref.LatestPipeline.ID, err.Error())
129+
return "", fmt.Errorf("could not fetch pipeline variables for %d: %s", pipeline.ID, err.Error())
130130
}
131131

132132
c.requestsRemaining(resp)

‎pkg/gitlab/pipelines_test.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,9 +78,12 @@ func TestGetRefPipelineVariablesAsConcatenatedString(t *testing.T) {
7878
Project: p,
7979
Name: "yay",
8080
}
81+
pipeline := schemas.Pipeline{
82+
ID: 1,
83+
}
8184

8285
// Should return right away as MostRecentPipeline is not defined
83-
variables, err := c.GetRefPipelineVariablesAsConcatenatedString(ctx, ref)
86+
variables, err := c.GetRefPipelineVariablesAsConcatenatedString(ctx, ref, schemas.Pipeline{})
8487
assert.NoError(t, err)
8588
assert.Equal(t, "", variables)
8689

@@ -89,14 +92,14 @@ func TestGetRefPipelineVariablesAsConcatenatedString(t *testing.T) {
8992
}
9093

9194
// Should fail as we have an invalid regexp pattern
92-
variables, err = c.GetRefPipelineVariablesAsConcatenatedString(ctx, ref)
95+
variables, err = c.GetRefPipelineVariablesAsConcatenatedString(ctx, ref, pipeline)
9396
assert.Error(t, err)
9497
assert.Contains(t, err.Error(), "the provided filter regex for pipeline variables is invalid")
9598
assert.Equal(t, "", variables)
9699

97100
// Should work
98101
ref.Project.Pull.Pipeline.Variables.Regexp = `.*`
99-
variables, err = c.GetRefPipelineVariablesAsConcatenatedString(ctx, ref)
102+
variables, err = c.GetRefPipelineVariablesAsConcatenatedString(ctx, ref, pipeline)
100103
assert.NoError(t, err)
101104
assert.Equal(t, "foo:bar,bar:baz", variables)
102105
}

‎pkg/schemas/metric.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
)
1010

1111
const (
12-
// MetricKindCoverage refers to the coerage of a job/pipeline.
12+
// MetricKindCoverage refers to the coverage of a job/pipeline.
1313
MetricKindCoverage MetricKind = iota
1414

1515
// MetricKindDurationSeconds ..
@@ -145,6 +145,7 @@ func (m Metric) Key() MetricKey {
145145
m.Labels["kind"],
146146
m.Labels["ref"],
147147
m.Labels["source"],
148+
m.Labels["variables"],
148149
})
149150

150151
case MetricKindJobArtifactSizeBytes, MetricKindJobDurationSeconds, MetricKindJobID, MetricKindJobQueuedDurationSeconds, MetricKindJobRunCount, MetricKindJobStatus, MetricKindJobTimestamp:

0 commit comments

Comments
 (0)