made the jobs queue bufferSize configurable

Author: mvisonneau

CHANGELOG.md

@@ -7,6 +7,30 @@ and this project adheres to [0ver](https://0ver.org) (more or less).
 
 ## [Unreleased]
 
+### Added
+
+- new metrics:
+  - `gitlab_ci_pipeline_test_report_total_time` -> Duration in seconds of all the tests in the most recently finished pipeline
+  - `gitlab_ci_pipeline_test_report_total_count` -> Number of total tests in the most recently finished pipeline
+  - `gitlab_ci_pipeline_test_report_success_count` -> Number of successful tests in the most recently finished pipeline
+  - `gitlab_ci_pipeline_test_report_failed_count` -> Number of failed tests in the most recently finished pipeline
+  - `gitlab_ci_pipeline_test_report_skipped_count` -> Number of skipped tests in the most recently finished pipeline
+  - `gitlab_ci_pipeline_test_report_error_count` -> Number of errored tests in the most recently finished pipeline
+  - `gitlab_ci_pipeline_test_suite_total_time` -> Duration in seconds for the test suite
+  - `gitlab_ci_pipeline_test_suite_total_count` -> Number of total tests for the test suite
+  - `gitlab_ci_pipeline_test_suite_success_count` -> Number of successful tests for the test suite
+  - `gitlab_ci_pipeline_test_suite_failed_count` -> Number of failed tests for the test suite
+  - `gitlab_ci_pipeline_test_suite_skipped_count` -> Number of skipped tests for the test suite
+  - `gitlab_ci_pipeline_test_suite_error_count` -> Duration in errored tests for the test suite
+- new configuration parameter: `gitlab.burstable_requests_per_second`, introducing a burstable amount of API RPS
+- new configuration parameter: `gitlab.maximum_jobs_queue_size`, controlling the queue buffer size
+
+### Changed
+
+- Upgraded golang to **v1.20**
+- Upgraded most dependencies to their latest versions
+- Reduced the amount of data being pulled from the project list API calls
+
 ## [v0.5.4] - 2022-08-25
 
 ### Added

docs/configuration_syntax.md

@@ -94,6 +94,17 @@ gitlab:
   # (optional, default: 5)
   burstable_requests_per_second: 5
 
+  # Maximum amount of jobs to keep queue, if this limit is reached
+  # newly created ones will get dropped. As a best practice you should not change this value.
+  # Workarounds to avoid hitting the limit are:
+  # - increase polling intervals
+  # - increase API rate limit
+  # - reduce the amount of projects, refs, environments or metrics you are looking into
+  # - leverage webhooks instead of polling schedules
+  #
+  # (optional, default: 1000)
+  maximum_jobs_queue_size: 1000
+
 pull:
   projects_from_wildcards:
     # Whether to trigger a discovery or not when the

pkg/config/config.go

@@ -114,6 +114,16 @@ type Gitlab struct {
 
 	// Burstable limit for the GitLab API requests/sec
 	BurstableRequestsPerSecond int `default:"5" validate:"gte=1" yaml:"burstable_requests_per_second"`
+
+	// Maximum amount of jobs to keep queue, if this limit is reached
+	// newly created ones will get dropped. As a best practice you should not change this value.
+	// Workarounds to avoid hitting the limit are:
+	// - increase polling intervals
+	// - increase API rate limit
+	// - reduce the amount of projects, refs, environments or metrics you are looking into
+	// - leverage webhooks instead of polling schedules
+	//
+	MaximumJobsQueueSize int `default:"1000" validate:"gte=10" yaml:"maximum_jobs_queue_size"`
 }
 
 // Redis ..

pkg/config/config_test.go

@@ -24,6 +24,7 @@ func TestNew(t *testing.T) {
 	c.Gitlab.EnableTLSVerify = true
 	c.Gitlab.MaximumRequestsPerSecond = 1
 	c.Gitlab.BurstableRequestsPerSecond = 5
+	c.Gitlab.MaximumJobsQueueSize = 1000
 
 	c.Pull.ProjectsFromWildcards.OnInit = true
 	c.Pull.ProjectsFromWildcards.Scheduled = true

pkg/controller/controller.go

@@ -52,7 +52,7 @@ func New(ctx context.Context, cfg config.Config, version string) (c Controller,
 		return
 	}
 
-	c.TaskController = NewTaskController(ctx, c.Redis)
+	c.TaskController = NewTaskController(ctx, c.Redis, cfg.Gitlab.MaximumJobsQueueSize)
 	c.registerTasks()
 
 	c.Store = store.New(ctx, c.Redis, c.Config.Projects)

pkg/controller/scheduler.go

@@ -19,8 +19,6 @@ import (
 	"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/store"
 )
 
-const bufferSize = 1000
-
 // TaskController holds task related clients.
 type TaskController struct {
 	Factory                  taskq.Factory
@@ -30,7 +28,7 @@ type TaskController struct {
 }
 
 // NewTaskController initializes and returns a new TaskController object.
-func NewTaskController(ctx context.Context, r *redis.Client) (t TaskController) {
+func NewTaskController(ctx context.Context, r *redis.Client, maximumJobsQueueSize int) (t TaskController) {
 	ctx, span := otel.Tracer(tracerName).Start(ctx, "controller:NewTaskController")
 	defer span.End()
 
@@ -40,7 +38,7 @@ func NewTaskController(ctx context.Context, r *redis.Client) (t TaskController)
 		Name:                 "default",
 		PauseErrorsThreshold: 3,
 		Handler:              t.TaskMap,
-		BufferSize:           bufferSize,
+		BufferSize:           maximumJobsQueueSize,
 	}
 
 	if r != nil {