Skip to content

Commit 4f8be76

Browse files
deeb00Semyon Koshel
deeb00
authored and
Semyon Koshel
committed
feat(opts): added delay before collecting instance metrics to avoud cloudwatch API throttling
Signed-off-by: Semyon Koshel <deathmetal1414@gmail.com>
1 parent a55f0e2 commit 4f8be76

6 files changed

Lines changed: 80 additions & 70 deletions

File tree

‎README.md‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -215,25 +215,26 @@ Prometheus RDS exporter</br>
215215

216216
Configuration could be defined in [prometheus-rds-exporter.yaml](https://github.com/qonto/prometheus-rds-exporter/blob/main/configs/prometheus-rds-exporter/prometheus-rds-exporter.yaml) or environment variables (format `PROMETHEUS_RDS_EXPORTER_<PARAMETER_NAME>`).
217217

218-
|Parameter | Description | Default |
219-
| ------------------------ | -------------------------------------------------------------------------------------------------------------------------- | ----------------------- |
220-
| aws-assume-role-arn | AWS IAM ARN role to assume to fetch metrics | |
221-
| aws-assume-role-session | AWS assume role session name | prometheus-rds-exporter |
222-
| collect-instance-metrics | Collect AWS instances metrics (AWS Cloudwatch API) | true |
223-
| collect-instance-tags | Collect AWS RDS tags | true |
224-
| collect-instance-types | Collect AWS instance types information (AWS EC2 API) | true |
225-
| collect-logs-size | Collect AWS instances logs size (AWS RDS API) | true |
226-
| collect-maintenances | Collect AWS instances maintenances (AWS RDS API) | true |
227-
| collect-quotas | Collect AWS RDS quotas (AWS quotas API) | true |
228-
| collect-usages | Collect AWS RDS usages (AWS Cloudwatch API) | true |
229-
| tag-selections | Tags to select database instances with. Refer to [dedicated section on tag configuration](#tag-configuration) | |
230-
| debug | Enable debug mode | |
231-
| enable-otel-traces | Enable OpenTelemetry traces. See [configuration](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/) | false |
232-
| listen-address | Address to listen on for web interface | :9043 |
233-
| log-format | Log format (`text` or `json`) | json |
234-
| metrics-path | Path under which to expose metrics | /metrics |
235-
| tls-cert-path | Path to TLS certificate | |
236-
| tls-key-path | Path to private key for TLS | |
218+
|Parameter | Description | Default |
219+
| ------------------------ |-----------------------------------------------------------------------------------------------------------------------------| ---------------------- |
220+
| aws-assume-role-arn | AWS IAM ARN role to assume to fetch metrics | |
221+
| aws-assume-role-session | AWS assume role session name | prometheus-rds-exporter |
222+
| collect-instance-metrics | Collect AWS instances metrics (AWS Cloudwatch API) | true |
223+
| collect-instance-metrics-delay" | Delay in collecting instance metrics (in seconds) to avoid CloudWatch API throttling. | 0 |
224+
| collect-instance-tags | Collect AWS RDS tags | true |
225+
| collect-instance-types | Collect AWS instance types information (AWS EC2 API) | true |
226+
| collect-logs-size | Collect AWS instances logs size (AWS RDS API) | true |
227+
| collect-maintenances | Collect AWS instances maintenances (AWS RDS API) | true |
228+
| collect-quotas | Collect AWS RDS quotas (AWS quotas API) | true |
229+
| collect-usages | Collect AWS RDS usages (AWS Cloudwatch API) | true |
230+
| tag-selections | Tags to select database instances with. Refer to [dedicated section on tag configuration](#tag-configuration) | |
231+
| debug | Enable debug mode | |
232+
| enable-otel-traces | Enable OpenTelemetry traces. See [configuration](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/) | false |
233+
| listen-address | Address to listen on for web interface | :9043 |
234+
| log-format | Log format (`text` or `json`) | json |
235+
| metrics-path | Path under which to expose metrics | /metrics |
236+
| tls-cert-path | Path to TLS certificate | |
237+
| tls-key-path | Path to private key for TLS | |
237238

238239
Configuration parameters priorities:
239240

‎cmd/root.go‎

Lines changed: 28 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -39,23 +39,24 @@ var (
3939
)
4040

4141
type exporterConfig struct {
42-
Debug bool `koanf:"debug"`
43-
LogFormat string `koanf:"log-format"`
44-
TLSCertPath string `koanf:"tls-cert-path"`
45-
TLSKeyPath string `koanf:"tls-key-path"`
46-
MetricPath string `koanf:"metrics-path"`
47-
ListenAddress string `koanf:"listen-address"`
48-
AWSAssumeRoleSession string `koanf:"aws-assume-role-session"`
49-
AWSAssumeRoleArn string `koanf:"aws-assume-role-arn"`
50-
CollectInstanceMetrics bool `koanf:"collect-instance-metrics"`
51-
CollectInstanceTags bool `koanf:"collect-instance-tags"`
52-
CollectInstanceTypes bool `koanf:"collect-instance-types"`
53-
CollectLogsSize bool `koanf:"collect-logs-size"`
54-
CollectMaintenances bool `koanf:"collect-maintenances"`
55-
CollectQuotas bool `koanf:"collect-quotas"`
56-
CollectUsages bool `koanf:"collect-usages"`
57-
OTELTracesEnabled bool `koanf:"enable-otel-traces"`
58-
TagSelections map[string][]string `koanf:"tag-selections"`
42+
Debug bool `koanf:"debug"`
43+
LogFormat string `koanf:"log-format"`
44+
TLSCertPath string `koanf:"tls-cert-path"`
45+
TLSKeyPath string `koanf:"tls-key-path"`
46+
MetricPath string `koanf:"metrics-path"`
47+
ListenAddress string `koanf:"listen-address"`
48+
AWSAssumeRoleSession string `koanf:"aws-assume-role-session"`
49+
AWSAssumeRoleArn string `koanf:"aws-assume-role-arn"`
50+
CollectInstanceMetrics bool `koanf:"collect-instance-metrics"`
51+
CollectInstanceMetricsDelay int `koanf:"collect-instance-metrics-delay"`
52+
CollectInstanceTags bool `koanf:"collect-instance-tags"`
53+
CollectInstanceTypes bool `koanf:"collect-instance-types"`
54+
CollectLogsSize bool `koanf:"collect-logs-size"`
55+
CollectMaintenances bool `koanf:"collect-maintenances"`
56+
CollectQuotas bool `koanf:"collect-quotas"`
57+
CollectUsages bool `koanf:"collect-usages"`
58+
OTELTracesEnabled bool `koanf:"enable-otel-traces"`
59+
TagSelections map[string][]string `koanf:"tag-selections"`
5960
}
6061

6162
func run(configuration exporterConfig) {
@@ -92,14 +93,15 @@ func run(configuration exporterConfig) {
9293
servicequotasClient := servicequotas.NewFromConfig(cfg)
9394

9495
collectorConfiguration := exporter.Configuration{
95-
CollectInstanceMetrics: configuration.CollectInstanceMetrics,
96-
CollectInstanceTypes: configuration.CollectInstanceTypes,
97-
CollectInstanceTags: configuration.CollectInstanceTags,
98-
CollectLogsSize: configuration.CollectLogsSize,
99-
CollectMaintenances: configuration.CollectMaintenances,
100-
CollectQuotas: configuration.CollectQuotas,
101-
CollectUsages: configuration.CollectUsages,
102-
TagSelections: configuration.TagSelections,
96+
CollectInstanceMetrics: configuration.CollectInstanceMetrics,
97+
CollectInstanceMetricsDelay: configuration.CollectInstanceMetricsDelay,
98+
CollectInstanceTypes: configuration.CollectInstanceTypes,
99+
CollectInstanceTags: configuration.CollectInstanceTags,
100+
CollectLogsSize: configuration.CollectLogsSize,
101+
CollectMaintenances: configuration.CollectMaintenances,
102+
CollectQuotas: configuration.CollectQuotas,
103+
CollectUsages: configuration.CollectUsages,
104+
TagSelections: configuration.TagSelections,
103105
}
104106

105107
collector := exporter.NewCollector(*logger, collectorConfiguration, awsAccountID, awsRegion, rdsClient, ec2Client, cloudWatchClient, servicequotasClient, tagClient)
@@ -163,6 +165,7 @@ func NewRootCommand() (*cobra.Command, error) {
163165
cmd.Flags().BoolP("collect-instance-tags", "", true, "Collect AWS RDS tags")
164166
cmd.Flags().BoolP("collect-instance-types", "", true, "Collect AWS instance types")
165167
cmd.Flags().BoolP("collect-instance-metrics", "", true, "Collect AWS instance metrics")
168+
cmd.Flags().IntP("collect-instance-metrics-delay", "", 0, "Delay in seconds before collecting instance metrics")
166169
cmd.Flags().BoolP("collect-logs-size", "", true, "Collect AWS instances logs size")
167170
cmd.Flags().BoolP("collect-maintenances", "", true, "Collect AWS instances maintenances")
168171
cmd.Flags().BoolP("collect-quotas", "", true, "Collect AWS RDS quotas")

‎configs/prometheus-rds-exporter/prometheus-rds-exporter.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@
4141
# Collect AWS instances metrics (AWS Cloudwatch API)
4242
# collect-instance-metrics: true
4343

44+
# Delay in seconds before collecting instance metrics (default: 0)
45+
# collect-instance-metrics-delay: 60
46+
4447
# Collect AWS instance tags (AWS RDS API)
4548
# collect-instance-tags: true
4649

‎internal/app/cloudwatch/rds.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -279,12 +279,12 @@ func (c *RdsFetcher) updateMetricsWithCloudWatchQueriesResult(metrics map[string
279279
return nil
280280
}
281281

282-
func (c *RdsFetcher) GetRDSInstanceMetrics(dbIdentifiers []string) (CloudWatchMetrics, error) {
282+
func (c *RdsFetcher) GetRDSInstanceMetrics(dbIdentifiers []string, delay int) (CloudWatchMetrics, error) {
283283
metrics := make(map[string]*RdsMetrics)
284284

285285
cloudWatchQueries := generateCloudWatchQueriesForInstances(dbIdentifiers)
286-
startTime := aws.Time(time.Now().Add(-5 * time.Minute)) // Start time - 5 minutes ago
287-
endTime := aws.Time(time.Now()) // End time - now
286+
startTime := aws.Time(time.Now().Add(-time.Duration(300+delay) * time.Second)) // Start time - 5 minutes ago + delay
287+
endTime := aws.Time(time.Now().Add(-time.Duration(delay) * time.Second)) // End time - now + delay
288288
chunkSize := MaxQueriesPerCloudwatchRequest
289289

290290
cloudWatchAPICalls := float64(0)

‎internal/app/exporter/exporter.go‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,15 @@ const (
3030
var tracer = otel.Tracer("github/qonto/prometheus-rds-exporter/internal/app/exporter")
3131

3232
type Configuration struct {
33-
CollectInstanceMetrics bool
34-
CollectInstanceTags bool
35-
CollectInstanceTypes bool
36-
CollectLogsSize bool
37-
CollectMaintenances bool
38-
CollectQuotas bool
39-
CollectUsages bool
40-
TagSelections map[string][]string
33+
CollectInstanceMetrics bool
34+
CollectInstanceMetricsDelay int
35+
CollectInstanceTags bool
36+
CollectInstanceTypes bool
37+
CollectLogsSize bool
38+
CollectMaintenances bool
39+
CollectQuotas bool
40+
CollectUsages bool
41+
TagSelections map[string][]string
4142
}
4243

4344
type counters struct {
@@ -520,7 +521,7 @@ func (c *rdsCollector) getCloudwatchMetrics(client cloudwatch.CloudWatchClient,
520521

521522
fetcher := cloudwatch.NewRDSFetcher(client, c.logger)
522523

523-
metrics, err := fetcher.GetRDSInstanceMetrics(instanceIdentifiers)
524+
metrics, err := fetcher.GetRDSInstanceMetrics(instanceIdentifiers, c.configuration.CollectInstanceMetricsDelay)
524525
if err != nil {
525526
c.counters.Errors++
526527
}

‎internal/app/exporter/exporter_test.go‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,14 @@ func TestWithAllDisabledCollectors(t *testing.T) {
3131
servicequotasClient := servicequotas_mock.ServiceQuotasClient{}
3232

3333
configuration := exporter.Configuration{
34-
CollectInstanceMetrics: false,
35-
CollectInstanceTypes: false,
36-
CollectInstanceTags: false,
37-
CollectLogsSize: false,
38-
CollectMaintenances: false,
39-
CollectQuotas: false,
40-
CollectUsages: false,
34+
CollectInstanceMetrics: false,
35+
CollectInstanceMetricsDelay: 0,
36+
CollectInstanceTypes: false,
37+
CollectInstanceTags: false,
38+
CollectLogsSize: false,
39+
CollectMaintenances: false,
40+
CollectQuotas: false,
41+
CollectUsages: false,
4142
}
4243

4344
collector := exporter.NewCollector(*logger, configuration, awsAccountID, awsRegion, rdsClient, ec2Client, cloudWatchClient, servicequotasClient, nil)
@@ -67,13 +68,14 @@ func TestCollector(t *testing.T) {
6768
servicequotasClient := servicequotas_mock.ServiceQuotasClient{}
6869

6970
configuration := exporter.Configuration{
70-
CollectInstanceMetrics: true,
71-
CollectInstanceTypes: true,
72-
CollectInstanceTags: false,
73-
CollectLogsSize: true,
74-
CollectMaintenances: true,
75-
CollectQuotas: true,
76-
CollectUsages: true,
71+
CollectInstanceMetrics: true,
72+
CollectInstanceMetricsDelay: 60,
73+
CollectInstanceTypes: true,
74+
CollectInstanceTags: false,
75+
CollectLogsSize: true,
76+
CollectMaintenances: true,
77+
CollectQuotas: true,
78+
CollectUsages: true,
7779
}
7880

7981
collector := exporter.NewCollector(*logger, configuration, awsAccountID, awsRegion, rdsClient, ec2Client, cloudWatchClient, servicequotasClient, nil)

0 commit comments

Comments
 (0)