Added Ex metrics

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/alicebob/miniredis/v2 v2.14.1
 	github.com/go-redis/redis/v8 v8.4.2
 	github.com/go-redis/redis_rate/v9 v9.1.0
+	github.com/gomodule/redigo v2.0.0+incompatible // indirect
 	github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40
 	github.com/mvisonneau/go-helpers v0.0.1
 	github.com/openlyinc/pointy v1.1.2
@@ -19,7 +20,10 @@ require (
 	github.com/vmihailenco/taskq/v3 v3.2.3
 	github.com/xanzy/go-gitlab v0.40.1
 	go.uber.org/ratelimit v0.1.0
+	gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
 )
 
 replace github.com/vmihailenco/taskq/v3 => github.com/mvisonneau/taskq/v3 v3.2.4-0.20201127170227-fddacd1811f5
+
+replace github.com/mvisonneau/gitlab-ci-pipelines-exporter => github.com/devopsext/gitlab-ci-pipelines-exporter v0.4.6-0.20201216132931-2ada86ba9eac

go.sum

@@ -170,6 +170,9 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v1.8.3 h1:HR0kYDX2RJZvAup8CsiJwxB4dTCSC0AaUq6S4SiLwUc=
 github.com/gomodule/redigo v1.8.3/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0=
+github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
+github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/gomodule/redigo/redis v0.0.0-do-not-use h1:J7XIp6Kau0WoyT4JtXHT3Ei0gA1KkSc6bc87j9v9WIo=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=

pkg/exporter/collectors.go

@@ -230,3 +230,47 @@ func NewCollectorRunCount() prometheus.Collector {
 		defaultLabels,
 	)
 }
+
+// NewCollectorExPipelineStatus returns a new collector for the gitlab_ci_ex_pipeline_status metric
+func NewCollectorExPipelineStatus() prometheus.Collector {
+	return prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "gitlab_ci_ex_pipeline_status",
+			Help: "Status of the most recent pipelines",
+		},
+		append(defaultLabels, "status", "pipeline_id"),
+	)
+}
+
+// NewCollectorExPipelineDuration returns a new collector for the gitlab_ci_ex_pipeline_duration metric
+func NewCollectorExPipelineDuration() prometheus.Collector {
+	return prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "gitlab_ci_ex_pipeline_duration",
+			Help: "Duration of the most recent pipelines",
+		},
+		append(defaultLabels, "pipeline_id"),
+	)
+}
+
+// NewCollectorExPipelineJobStatus returns a new collector for the gitlab_ci_ex_pipeline_job_status metric
+func NewCollectorExPipelineJobStatus() prometheus.Collector {
+	return prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "gitlab_ci_ex_pipeline_job_status",
+			Help: "Status of the most recent pipelines",
+		},
+		append(defaultLabels, append(jobLabels, "status", "pipeline_id", "job_id")...),
+	)
+}
+
+// NewCollectorExPipelineJobDuration returns a new collector for the gitlab_ci_ex_pipeline_job_duration metric
+func NewCollectorExPipelineJobDuration() prometheus.Collector {
+	return prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "gitlab_ci_ex_pipeline_job_duration",
+			Help: "Duration of the most recent pipelines",
+		},
+		append(defaultLabels, append(jobLabels, "pipeline_id", "job_id")...),
+	)
+}

pkg/exporter/jobs.go

@@ -3,6 +3,7 @@ package exporter
 import (
 	"reflect"
 	"regexp"
+	"strconv"
 
 	"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas"
 	log "github.com/sirupsen/logrus"
@@ -149,3 +150,69 @@ func processJobMetrics(ref schemas.Ref, job schemas.Job) {
 		ref.OutputSparseStatusMetrics,
 	)
 }
+
+func pullRefExPipelineJobsMetrics(ref schemas.Ref, pipelineID int) error {
+	cfgUpdateLock.RLock()
+	defer cfgUpdateLock.RUnlock()
+
+	jobs, err := gitlabClient.ListRefExPipelineJobs(ref, pipelineID)
+	if err != nil {
+		return err
+	}
+
+	for _, job := range jobs {
+		processExJobMetrics(ref, job, pipelineID)
+	}
+
+	return nil
+}
+
+func processExJobMetrics(ref schemas.Ref, job schemas.Job, pipelineID int) {
+	cfgUpdateLock.RLock()
+	defer cfgUpdateLock.RUnlock()
+
+	labels := ref.DefaultLabelsValues()
+	labels["stage"] = job.Stage
+	labels["job_name"] = job.Name
+	labels["runner_description"] = job.Runner.Description
+	labels["pipeline_id"] = strconv.Itoa(pipelineID)
+	labels["job_id"] = strconv.Itoa(job.ID)
+
+	projectRefLogFields := log.Fields{
+		"project-name": ref.ProjectName,
+		"job-name":     job.Name,
+		"job-id":       job.ID,
+	}
+
+	// Refresh ref state from the store
+	if err := store.GetRef(&ref); err != nil {
+		log.WithFields(
+			projectRefLogFields,
+		).WithField("error", err.Error()).Error("getting ref from the store")
+		return
+	}
+
+	if err := store.SetRef(ref); err != nil {
+		log.WithFields(
+			projectRefLogFields,
+		).WithField("error", err.Error()).Error("writing ref in the store")
+		return
+	}
+
+	log.WithFields(projectRefLogFields).Debug("processing job metrics")
+
+	emitStatusMetric(
+		schemas.MetricKindExPipelineJobStatus,
+		labels,
+		statusesList[:],
+		job.Status,
+		ref.OutputSparseStatusMetrics,
+	)
+
+	storeSetMetric(schemas.Metric{
+		Kind:   schemas.MetricKindExPipelineJobDuration,
+		Labels: labels,
+		Value:  job.DurationSeconds,
+	})
+
+}

pkg/exporter/metrics.go

@@ -46,6 +46,10 @@ func NewRegistry() *Registry {
 			schemas.MetricKindRunCount:                             NewCollectorRunCount(),
 			schemas.MetricKindStatus:                               NewCollectorStatus(),
 			schemas.MetricKindTimestamp:                            NewCollectorTimestamp(),
+			schemas.MetricKindExPipelineStatus:                     NewCollectorExPipelineStatus(),
+			schemas.MetricKindExPipelineDuration:                   NewCollectorExPipelineDuration(),
+			schemas.MetricKindExPipelineJobStatus:                  NewCollectorExPipelineJobStatus(),
+			schemas.MetricKindExPipelineJobDuration:                NewCollectorExPipelineJobDuration(),
 		},
 	}
 

pkg/exporter/pipelines.go

@@ -3,6 +3,7 @@ package exporter
 import (
 	"fmt"
 	"reflect"
+	"strconv"
 
 	"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas"
 	log "github.com/sirupsen/logrus"
@@ -39,7 +40,7 @@ func pullRefMetrics(ref schemas.Ref) error {
 	pipelines, err := gitlabClient.GetProjectPipelines(ref.ProjectName, &goGitlab.ListProjectPipelinesOptions{
 		// We only need the most recent pipeline
 		ListOptions: goGitlab.ListOptions{
-			PerPage: 1,
+			PerPage: ref.PullPipelineDepth,
 			Page:    1,
 		},
 		Ref: goGitlab.String(ref.Name),
@@ -54,6 +55,7 @@ func pullRefMetrics(ref schemas.Ref) error {
 		return nil
 	}
 
+	// Getting metrics for the most recent pipeline
 	pipeline, err := gitlabClient.GetRefPipeline(ref, pipelines[0].ID)
 	if err != nil {
 		return err
@@ -135,5 +137,37 @@ func pullRefMetrics(ref schemas.Ref) error {
 		}
 	}
 
+	// Getting metrics for the other pipelines (Ex metrics)
+	for i := range pipelines {
+
+		pipeline, err := gitlabClient.GetRefPipeline(ref, pipelines[i].ID)
+		if err != nil {
+			return err
+		}
+
+		labels := ref.DefaultLabelsValues()
+		labels["pipeline_id"] = strconv.Itoa(pipeline.ID)
+
+		emitStatusMetric(
+			schemas.MetricKindExPipelineStatus,
+			labels,
+			statusesList[:],
+			pipeline.Status,
+			ref.OutputSparseStatusMetrics,
+		)
+
+		storeSetMetric(schemas.Metric{
+			Kind:   schemas.MetricKindExPipelineDuration,
+			Labels: labels,
+			Value:  pipeline.DurationSeconds,
+		})
+
+		if ref.PullPipelineJobsEnabled {
+			if err := pullRefExPipelineJobsMetrics(ref, pipelines[i].ID); err != nil {
+				return err
+			}
+		}
+	}
+
 	return nil
 }

pkg/exporter/refs.go

@@ -85,6 +85,7 @@ func pullRefsFromProject(p schemas.Project) error {
 			p.Pull.Pipeline.Jobs.FromChildPipelines.Enabled(),
 			p.Pull.Pipeline.Variables.Enabled(),
 			p.Pull.Pipeline.Variables.Regexp(),
+			p.Pull.Pipeline.Depth(),
 		)
 
 		refExists, err := store.RefExists(ref.Key())

pkg/gitlab/jobs.go

@@ -227,3 +227,26 @@ func (c *Client) ListRefMostRecentJobs(ref schemas.Ref) (jobs []schemas.Job, err
 	}
 	return
 }
+
+// ListRefExPipelineJobs ..
+func (c *Client) ListRefExPipelineJobs(ref schemas.Ref, pipelineID int) (jobs []schemas.Job, err error) {
+
+	jobs, err = c.ListPipelineJobs(ref.ProjectName, pipelineID)
+	if err != nil {
+		return
+	}
+
+	if ref.PullPipelineJobsFromChildPipelinesEnabled {
+		var childJobs []schemas.Job
+		childJobs, err = c.ListPipelineChildJobs(ref.ProjectName, pipelineID)
+		if err != nil {
+			return
+		}
+
+		for _, childJob := range childJobs {
+			jobs = append(jobs, childJob)
+		}
+	}
+
+	return
+}

pkg/gitlab/pipelines.go

@@ -216,6 +216,7 @@ func (c *Client) GetRefsFromPipelines(p schemas.Project, topics string) (schemas
 					p.Pull.Pipeline.Jobs.FromChildPipelines.Enabled(),
 					p.Pull.Pipeline.Variables.Enabled(),
 					p.Pull.Pipeline.Variables.Regexp(),
+					p.Pull.Pipeline.Depth(),
 				)
 
 				if _, ok := refs[ref.Key()]; !ok {

pkg/schemas/metric.go

@@ -68,6 +68,18 @@ const (
 
 	// MetricKindTimestamp ..
 	MetricKindTimestamp
+
+	// MetricKindExPipelineStatus ..
+	MetricKindExPipelineStatus
+
+	// MetricKindExPipelineDuration ..
+	MetricKindExPipelineDuration
+
+	// MetricKindExPipelineJobStatus ..
+	MetricKindExPipelineJobStatus
+
+	// MetricKindExPipelineJobDuration ..
+	MetricKindExPipelineJobDuration
 )
 
 // MetricKind ..
@@ -120,5 +132,33 @@ func (m Metric) Key() MetricKey {
 		key += m.Labels["status"]
 	}
 
+	// Ex metrics
+	switch m.Kind {
+	case MetricKindExPipelineStatus, MetricKindExPipelineDuration:
+		key += fmt.Sprintf("%v", []string{
+			m.Labels["project"],
+			m.Labels["kind"],
+			m.Labels["ref"],
+			m.Labels["topics"],
+			m.Labels["variables"],
+			m.Labels["status"],
+			m.Labels["pipeline_id"],
+		})
+	case MetricKindExPipelineJobStatus, MetricKindExPipelineJobDuration:
+		key += fmt.Sprintf("%v", []string{
+			m.Labels["project"],
+			m.Labels["kind"],
+			m.Labels["ref"],
+			m.Labels["stage"],
+			m.Labels["job_name"],
+			m.Labels["runner_description"],
+			m.Labels["topics"],
+			m.Labels["variables"],
+			m.Labels["status"],
+			m.Labels["pipeline_id"],
+			m.Labels["job_id"],
+		})
+	}
+
 	return MetricKey(strconv.Itoa(int(crc32.ChecksumIEEE([]byte(key)))))
 }

pkg/schemas/project.go

@@ -20,6 +20,7 @@ var (
 	defaultProjectPullPipelineJobsFromChildPipelinesEnabled      = true
 	defaultProjectPullPipelineVariablesEnabled                   = false
 	defaultProjectPullPipelineVariablesRegexp                    = `.*`
+	defaultProjectPullPipelinesDepth                             = 10
 )
 
 // ProjectParameters for the fetching configuration of Projects and Wildcards
@@ -85,8 +86,9 @@ type ProjectPullRefsFromMergeRequests ProjectPullRefsFromParameters
 
 // ProjectPullPipeline ..
 type ProjectPullPipeline struct {
-	Jobs      ProjectPullPipelineJobs      `yaml:"jobs"`
-	Variables ProjectPullPipelineVariables `yaml:"variables"`
+	Jobs       ProjectPullPipelineJobs      `yaml:"jobs"`
+	Variables  ProjectPullPipelineVariables `yaml:"variables"`
+	DepthValue *int                         `yaml:"depth"`
 }
 
 // ProjectPullPipelineJobs ..
@@ -166,6 +168,10 @@ func UpdateProjectDefaults(d ProjectParameters) {
 	if d.Pull.Pipeline.Variables.RegexpValue != nil {
 		defaultProjectPullPipelineVariablesRegexp = *d.Pull.Pipeline.Variables.RegexpValue
 	}
+
+	if d.Pull.Pipeline.DepthValue != nil {
+		defaultProjectPullPipelinesDepth = *d.Pull.Pipeline.DepthValue
+	}
 }
 
 // Project holds information about a GitLab project
@@ -313,3 +319,12 @@ func (p *ProjectPullPipelineVariables) Regexp() string {
 
 	return defaultProjectPullPipelineVariablesRegexp
 }
+
+// Depth ...
+func (p *ProjectPullPipeline) Depth() int {
+	if p.DepthValue != nil {
+		return *p.DepthValue
+	}
+
+	return defaultProjectPullPipelinesDepth
+}

pkg/schemas/ref.go

@@ -34,6 +34,7 @@ type Ref struct {
 	PullPipelineJobsFromChildPipelinesEnabled bool
 	PullPipelineVariablesEnabled              bool
 	PullPipelineVariablesRegexp               string
+	PullPipelineDepth                         int
 }
 
 // RefKey ..
@@ -70,6 +71,7 @@ func NewRef(
 	projectName, name, topics string,
 	outputSparseStatusMetrics, pullPipelineJobsEnabled, pullPipelineJobsFromChildPipelinesEnabled, pullPipelineVariablesEnabled bool,
 	pullPipelineVariablesRegexp string,
+	pullPipelineDepth int,
 ) Ref {
 	return Ref{
 		Kind:        kind,
@@ -83,5 +85,6 @@ func NewRef(
 		PullPipelineJobsFromChildPipelinesEnabled: pullPipelineJobsFromChildPipelinesEnabled,
 		PullPipelineVariablesEnabled:              pullPipelineVariablesEnabled,
 		PullPipelineVariablesRegexp:               pullPipelineVariablesRegexp,
+		PullPipelineDepth:                         pullPipelineDepth,
 	}
 }

pkg/schemas/ref_test.go

@@ -71,5 +71,6 @@ func TestNewRef(t *testing.T) {
 		false,
 		true,
 		".*",
+		1,
 	))
 }