Skip to content

Commit aacec27

Browse files
mblaschkemblaschke
committed
add servicediscovery cache
Signed-off-by: Markus Blaschke <[email protected]>
1 parent 8c2b453 commit aacec27

File tree

6 files changed

+108
-35
lines changed

6 files changed

+108
-35
lines changed

‎README.md‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,18 @@ Application Options:
2323
--debug debug mode [$DEBUG]
2424
-v, --verbose verbose mode [$VERBOSE]
2525
--log.json Switch log output to json format [$LOG_JSON]
26-
--azure-environment= Azure environment name (default: AZUREPUBLICCLOUD) [$AZURE_ENVIRONMENT]
27-
--azure-ad-resource-url= Specifies the AAD resource ID to use. If not set, it defaults to ResourceManagerEndpoint for operations with Azure Resource Manager [$AZURE_AD_RESOURCE]
28-
--concurrency.subscription= Concurrent subscription fetches (default: 5) [$CONCURRENCY_SUBSCRIPTION]
29-
--concurrency.subscription.resource= Concurrent requests per resource (inside subscription requests) (default:
30-
10) [$CONCURRENCY_SUBSCRIPTION_RESOURCE]
26+
--azure-environment= Azure environment name (default: AZUREPUBLICCLOUD)
27+
[$AZURE_ENVIRONMENT]
28+
--azure-ad-resource-url= Specifies the AAD resource ID to use. If not set, it defaults to
29+
ResourceManagerEndpoint for operations with Azure Resource Manager
30+
[$AZURE_AD_RESOURCE]
31+
--azure.servicediscovery.cache= Duration for caching Azure ServiceDiscovery of workspaces to reduce
32+
API calls (time.Duration) (default: 30m)
33+
[$AZURE_SERVICEDISCOVERY_CACHE]
34+
--concurrency.subscription= Concurrent subscription fetches (default: 5)
35+
[$CONCURRENCY_SUBSCRIPTION]
36+
--concurrency.subscription.resource= Concurrent requests per resource (inside subscription requests)
37+
(default: 10) [$CONCURRENCY_SUBSCRIPTION_RESOURCE]
3138
--enable-caching Enable internal caching [$ENABLE_CACHING]
3239
--bind= Server address (default: :8080) [$SERVER_BIND]
3340

‎azure_insights.go‎

Lines changed: 82 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,29 +2,41 @@ package main
22

33
import (
44
"context"
5+
"crypto/sha1"
6+
"encoding/json"
7+
"fmt"
58
"github.com/Azure/azure-sdk-for-go/profiles/latest/resources/mgmt/resources"
69
"github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2018-03-01/insights"
710
"github.com/Azure/go-autorest/autorest"
811
"github.com/prometheus/client_golang/prometheus"
12+
log "github.com/sirupsen/logrus"
913
prometheusCommon "github.com/webdevops/go-prometheus-common"
1014
"net/http"
1115
"strconv"
1216
"strings"
17+
"time"
1318
)
1419

15-
type AzureInsightMetrics struct {
16-
authorizer *autorest.Authorizer
17-
prometheusRegistry *prometheus.Registry
20+
type (
21+
AzureInsightMetrics struct {
22+
authorizer *autorest.Authorizer
23+
prometheusRegistry *prometheus.Registry
1824

19-
prometheus struct {
20-
apiQuota *prometheus.GaugeVec
25+
prometheus struct {
26+
apiQuota *prometheus.GaugeVec
27+
}
2128
}
22-
}
2329

24-
type AzureInsightMetricsResult struct {
25-
Result *insights.Response
26-
ResourceID *string
27-
}
30+
AzureInsightMetricsResult struct {
31+
Result *insights.Response
32+
ResourceID *string
33+
}
34+
35+
AzureResource struct {
36+
ID *string
37+
Tags map[string]*string
38+
}
39+
)
2840

2941
func NewAzureInsightMetrics(authorizer autorest.Authorizer, registry *prometheus.Registry) *AzureInsightMetrics {
3042
ret := AzureInsightMetrics{}
@@ -87,8 +99,66 @@ func (m *AzureInsightMetrics) azureResponseInsepector(subscriptionId string) aut
8799
}
88100
}
89101

90-
func (m *AzureInsightMetrics) ListResources(subscriptionId, filter string) (resources.ListResultIterator, error) {
91-
return m.ResourcesClient(subscriptionId).ListComplete(context.Background(), filter, "", nil)
102+
func (m *AzureInsightMetrics) ListResources(ctx context.Context, logger *log.Entry, subscriptionId, filter string, w http.ResponseWriter) ([]AzureResource, error) {
103+
var cacheDuration *time.Duration
104+
cacheKey := ""
105+
106+
resourceList := []AzureResource{}
107+
108+
if opts.Azure.ServiceDiscovery.CacheDuration != nil && opts.Azure.ServiceDiscovery.CacheDuration.Seconds() > 0 {
109+
cacheDuration = opts.Azure.ServiceDiscovery.CacheDuration
110+
cacheKey = fmt.Sprintf(
111+
"sd:%x",
112+
string(sha1.New().Sum([]byte(fmt.Sprintf("%v:%v", subscriptionId, filter)))),
113+
)
114+
}
115+
// try cache
116+
if cacheDuration != nil {
117+
if v, ok := azureCache.Get(cacheKey); ok {
118+
if cacheData, ok := v.([]byte); ok {
119+
if err := json.Unmarshal(cacheData, &resourceList); err == nil {
120+
logger.Debug("fetched servicediscovery from cache")
121+
w.Header().Add("X-servicediscovery-cached", "true")
122+
return resourceList, nil
123+
} else {
124+
logger.Debug("unable to parse cached servicediscovery")
125+
}
126+
}
127+
}
128+
}
129+
130+
list, err := m.ResourcesClient(subscriptionId).ListComplete(context.Background(), filter, "", nil)
131+
if err != nil {
132+
return resourceList, err
133+
}
134+
135+
for list.NotDone() {
136+
val := list.Value()
137+
138+
resourceList = append(
139+
resourceList,
140+
AzureResource{
141+
ID: val.ID,
142+
Tags: val.Tags,
143+
},
144+
)
145+
146+
if list.NextWithContext(ctx) != nil {
147+
break
148+
}
149+
}
150+
151+
// store to cache (if enabeld)
152+
if cacheDuration != nil {
153+
logger.Debug("saving servicedisccovery to cache")
154+
if cacheData, err := json.Marshal(resourceList); err == nil {
155+
w.Header().Add("X-servicediscovery-cached-until", time.Now().Add(*cacheDuration).Format(time.RFC3339))
156+
azureCache.Set(cacheKey, cacheData, *cacheDuration)
157+
logger.Debugf("saved servicediscovery to cache for %s", cacheDuration.String())
158+
}
159+
}
160+
161+
return resourceList, nil
92162
}
93163

94164
func (m *AzureInsightMetrics) CreatePrometheusMetricsGauge(metricName string) (gauge *prometheus.GaugeVec) {

‎config/opts.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package config
33
import (
44
"encoding/json"
55
log "github.com/sirupsen/logrus"
6+
"time"
67
)
78

89
type (
@@ -16,8 +17,11 @@ type (
1617

1718
// azure
1819
Azure struct {
19-
Environment *string `long:"azure-environment" env:"AZURE_ENVIRONMENT" description:"Azure environment name" default:"AZUREPUBLICCLOUD"`
20-
AdResourceUrl *string `long:"azure-ad-resource-url" env:"AZURE_AD_RESOURCE" description:"Specifies the AAD resource ID to use. If not set, it defaults to ResourceManagerEndpoint for operations with Azure Resource Manager"`
20+
Environment *string `long:"azure-environment" env:"AZURE_ENVIRONMENT" description:"Azure environment name" default:"AZUREPUBLICCLOUD"`
21+
AdResourceUrl *string `long:"azure-ad-resource-url" env:"AZURE_AD_RESOURCE" description:"Specifies the AAD resource ID to use. If not set, it defaults to ResourceManagerEndpoint for operations with Azure Resource Manager"`
22+
ServiceDiscovery struct {
23+
CacheDuration *time.Duration `long:"azure.servicediscovery.cache" env:"AZURE_SERVICEDISCOVERY_CACHE" description:"Duration for caching Azure ServiceDiscovery of workspaces to reduce API calls (time.Duration)" default:"30m"`
24+
}
2125
}
2226

2327
// Prober settings

‎main.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ var (
5151
prometheusMetricRequests *prometheus.CounterVec
5252

5353
metricsCache *cache.Cache
54+
azureCache *cache.Cache
5455

5556
// Git version information
5657
gitCommit = "<unknown>"
@@ -63,6 +64,7 @@ func main() {
6364
log.Infof("starting azure-metrics-exporter v%s (%s; %s; by %v)", gitTag, gitCommit, runtime.Version(), Author)
6465
log.Info(string(opts.GetJson()))
6566
metricsCache = cache.New(1*time.Minute, 1*time.Minute)
67+
azureCache = cache.New(1*time.Minute, 1*time.Minute)
6668

6769
log.Infof("init Azure connection")
6870
initAzureConnection()

‎probe_metrics_list.go‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -62,16 +62,15 @@ func probeMetricsListHandler(w http.ResponseWriter, r *http.Request) {
6262
wgResource := sizedwaitgroup.New(opts.Prober.ConcurrencySubscriptionResource)
6363

6464
// fetch list of resources
65-
list, err := azureInsightMetrics.ListResources(subscription, settings.Filter)
66-
65+
list, err := azureInsightMetrics.ListResources(ctx, contextLogger, subscription, settings.Filter, w)
6766
if err != nil {
6867
contextLogger.Errorln(err)
6968
http.Error(w, err.Error(), http.StatusBadRequest)
7069
return
7170
}
7271

73-
for list.NotDone() {
74-
val := list.Value()
72+
for _, row := range list {
73+
val := row
7574

7675
wgResource.Add()
7776
go func() {
@@ -104,10 +103,6 @@ func probeMetricsListHandler(w http.ResponseWriter, r *http.Request) {
104103
}).Inc()
105104
}
106105
}()
107-
108-
if list.NextWithContext(ctx) != nil {
109-
break
110-
}
111106
}
112107

113108
wgResource.Wait()

‎probe_metrics_scrape.go‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -75,16 +75,15 @@ func probeMetricsScrapeHandler(w http.ResponseWriter, r *http.Request) {
7575
defer wg.Done()
7676
wgResource := sizedwaitgroup.New(opts.Prober.ConcurrencySubscriptionResource)
7777

78-
list, err := azureInsightMetrics.ListResources(subscription, settings.Filter)
79-
78+
list, err := azureInsightMetrics.ListResources(ctx, contextLogger, subscription, settings.Filter, w)
8079
if err != nil {
8180
contextLogger.Errorln(err)
8281
http.Error(w, err.Error(), http.StatusBadRequest)
8382
return
8483
}
8584

86-
for list.NotDone() {
87-
val := list.Value()
85+
for _, row := range list {
86+
val := row
8887

8988
resourceLogger := contextLogger.WithFields(log.Fields{
9089
"azureSubscription": subscription,
@@ -123,10 +122,6 @@ func probeMetricsScrapeHandler(w http.ResponseWriter, r *http.Request) {
123122
}
124123
}
125124
}()
126-
127-
if list.NextWithContext(ctx) != nil {
128-
break
129-
}
130125
}
131126

132127
wgResource.Wait()

0 commit comments

Comments
 (0)