added providerconfig labels to NEG metrics

Author: priyapande

cmd/glbc/main.go

@@ -59,18 +59,18 @@ import (
 	"k8s.io/ingress-gce/pkg/utils"
 	"k8s.io/klog/v2"
 
-	ingctx "k8s.io/ingress-gce/pkg/context"
-	"k8s.io/ingress-gce/pkg/controller"
-	"k8s.io/ingress-gce/pkg/neg"
-	"k8s.io/ingress-gce/pkg/neg/syncers/labels"
-	negtypes "k8s.io/ingress-gce/pkg/neg/types"
-
 	"k8s.io/ingress-gce/cmd/glbc/app"
 	"k8s.io/ingress-gce/pkg/backendconfig"
+	ingctx "k8s.io/ingress-gce/pkg/context"
+	"k8s.io/ingress-gce/pkg/controller"
 	"k8s.io/ingress-gce/pkg/crd"
 	"k8s.io/ingress-gce/pkg/firewalls"
 	"k8s.io/ingress-gce/pkg/flags"
 	_ "k8s.io/ingress-gce/pkg/klog"
+	"k8s.io/ingress-gce/pkg/neg"
+	"k8s.io/ingress-gce/pkg/neg/metrics"
+	"k8s.io/ingress-gce/pkg/neg/syncers/labels"
+	negtypes "k8s.io/ingress-gce/pkg/neg/types"
 	"k8s.io/ingress-gce/pkg/utils/zonegetter"
 	"k8s.io/ingress-gce/pkg/version"
 )
@@ -656,6 +656,8 @@ func createNEGController(ctx *ingctx.ControllerContext, systemHealth *systemheal
 		adapter = ctx.Cloud
 	}
 
+	negMetrics := metrics.NewNegMetrics("")
+
 	// TODO: Refactor NEG to use cloud mocks so ctx.Cloud can be referenced within NewController.
 	negController, err := neg.NewController(
 		ctx.KubeClient,
@@ -674,7 +676,7 @@ func createNEGController(ctx *ingctx.ControllerContext, systemHealth *systemheal
 		ctx.HasSynced,
 		ctx.L4Namer,
 		ctx.DefaultBackendSvcPort,
-		negtypes.NewAdapterWithRateLimitSpecs(ctx.Cloud, flags.F.GCERateLimit.Values(), adapter),
+		negtypes.NewAdapterWithRateLimitSpecs(ctx.Cloud, flags.F.GCERateLimit.Values(), adapter, negMetrics),
 		zoneGetter,
 		ctx.ClusterNamer,
 		flags.F.ResyncPeriod,
@@ -691,6 +693,7 @@ func createNEGController(ctx *ingctx.ControllerContext, systemHealth *systemheal
 		flags.F.ReadOnlyMode,
 		stopCh,
 		logger,
+		negMetrics,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create NEG controller: %w", err)

docs/deploy/local/README.md

@@ -32,21 +32,14 @@ from the root of this repo:
 $ hack/setup-local.sh <cluster-name>
 ```
 
-## Setup GCE permissions
+## Authorize gcloud and kubectl
 
-When running locally, the Ingress-GCE controller looks on the local machine
-for credentials to create GCE networking resources. Specifically it looks for a
-json file specified at the GOOGLE_APPLICATION_CREDENTIALS variable. Given this,
-it is most desirable to follow these steps:
-
-1. Create a Service Account in GCP and give the account Compute Admin permissions
-
-2. Create a key for the Service Account and download it
-
-Then run the following:
+Once the cluster is ready prepare authorization to it.
+You need to authorize both gcloud and kubectl.
 
 ```console
-export GOOGLE_APPLICATION_CREDENTIALS=/path/to/key/file
+$ gcloud auth application-default login
+$ gcloud container clusters get-credentials CLUSTER_NAME --region CLUSTER_LOCATION
 ```
 
 ## Run the controller
@@ -62,6 +55,15 @@ binary in a container and place it in `bin/amd64`.
 make build
 ```
 
+NOTE -
+If you get build errors with Docker getting a permission denied while pulling the base image run the following commands:
+
+```console
+sudo addgroup --system docker
+sudo usermod -aG docker $USER
+newgrp docker
+```
+
 For Mac OS users or to build the binary locally and output it in the
 `bin/amd64` directory run:
 

pkg/controller/controller.go

@@ -48,6 +48,7 @@ import (
 	"k8s.io/ingress-gce/pkg/loadbalancers"
 	"k8s.io/ingress-gce/pkg/loadbalancers/features"
 	"k8s.io/ingress-gce/pkg/metrics"
+	negmetrics "k8s.io/ingress-gce/pkg/neg/metrics"
 	negtypes "k8s.io/ingress-gce/pkg/neg/types"
 	ingsync "k8s.io/ingress-gce/pkg/sync"
 	"k8s.io/ingress-gce/pkg/translator"
@@ -139,7 +140,7 @@ func NewLoadBalancerController(
 		instancePool:                   ctx.InstancePool,
 		l7Pool:                         loadbalancers.NewLoadBalancerPool(ctx.Cloud, ctx.ClusterNamer, ctx, namer.NewFrontendNamerFactory(ctx.ClusterNamer, ctx.KubeSystemUID, logger), logger),
 		backendSyncer:                  backends.NewBackendSyncer(backendPool, healthChecker, ctx.Cloud, ctx.Translator),
-		negLinker:                      backends.NewNEGLinker(backendPool, negtypes.NewAdapter(ctx.Cloud), ctx.Cloud, ctx.SvcNegInformer.GetIndexer(), logger),
+		negLinker:                      backends.NewNEGLinker(backendPool, negtypes.NewAdapter(ctx.Cloud, negmetrics.NewNegMetrics("")), ctx.Cloud, ctx.SvcNegInformer.GetIndexer(), logger),
 		igLinker:                       backends.NewInstanceGroupLinker(ctx.InstancePool, backendPool, logger),
 		metrics:                        ctx.ControllerMetrics,
 		ZoneGetter:                     ctx.ZoneGetter,

pkg/l4lb/l4controller.go

@@ -38,6 +38,7 @@ import (
 	"k8s.io/ingress-gce/pkg/forwardingrules"
 	"k8s.io/ingress-gce/pkg/l4lb/metrics"
 	"k8s.io/ingress-gce/pkg/loadbalancers"
+	negmetrics "k8s.io/ingress-gce/pkg/neg/metrics"
 	negtypes "k8s.io/ingress-gce/pkg/neg/types"
 	"k8s.io/ingress-gce/pkg/network"
 	"k8s.io/ingress-gce/pkg/utils"
@@ -106,7 +107,7 @@ func NewILBController(ctx *context.ControllerContext, stopCh <-chan struct{}, lo
 		hasSynced:       ctx.HasSynced,
 	}
 	l4c.backendPool = backends.NewPool(ctx.Cloud, l4c.namer)
-	l4c.NegLinker = backends.NewNEGLinker(l4c.backendPool, negtypes.NewAdapter(ctx.Cloud), ctx.Cloud, ctx.SvcNegInformer.GetIndexer(), logger)
+	l4c.NegLinker = backends.NewNEGLinker(l4c.backendPool, negtypes.NewAdapter(ctx.Cloud, negmetrics.NewNegMetrics("")), ctx.Cloud, ctx.SvcNegInformer.GetIndexer(), logger)
 
 	l4c.svcQueue = utils.NewPeriodicTaskQueueWithMultipleWorkers("l4", "services", l4c.numWorkers, l4c.syncWrapper, logger)
 

pkg/l4lb/l4netlbcontroller.go

@@ -39,6 +39,7 @@ import (
 	"k8s.io/ingress-gce/pkg/instancegroups"
 	"k8s.io/ingress-gce/pkg/l4lb/metrics"
 	"k8s.io/ingress-gce/pkg/loadbalancers"
+	negmetrics "k8s.io/ingress-gce/pkg/neg/metrics"
 	negtypes "k8s.io/ingress-gce/pkg/neg/types"
 	"k8s.io/ingress-gce/pkg/network"
 	"k8s.io/ingress-gce/pkg/utils"
@@ -138,7 +139,7 @@ func NewL4NetLBController(
 		adapter = ctx.Cloud
 	}
 	l4netLBc.networkResolver = network.NewNetworksResolver(networkLister, gkeNetworkParamSetLister, adapter, ctx.EnableMultinetworking, logger)
-	l4netLBc.negLinker = backends.NewNEGLinker(l4netLBc.backendPool, negtypes.NewAdapter(ctx.Cloud), ctx.Cloud, ctx.SvcNegInformer.GetIndexer(), logger)
+	l4netLBc.negLinker = backends.NewNEGLinker(l4netLBc.backendPool, negtypes.NewAdapter(ctx.Cloud, negmetrics.NewNegMetrics("")), ctx.Cloud, ctx.SvcNegInformer.GetIndexer(), logger)
 	l4netLBc.svcQueue = utils.NewPeriodicTaskQueueWithMultipleWorkers("l4netLB", "services", ctx.NumL4NetLBWorkers, l4netLBc.syncWrapper, logger)
 
 	ctx.ServiceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{

pkg/multiproject/neg/neg.go

@@ -16,6 +16,7 @@ import (
 	"k8s.io/ingress-gce/pkg/flags"
 	"k8s.io/ingress-gce/pkg/multiproject/filteredinformer"
 	"k8s.io/ingress-gce/pkg/neg"
+	"k8s.io/ingress-gce/pkg/neg/metrics"
 	"k8s.io/ingress-gce/pkg/neg/syncers/labels"
 	negtypes "k8s.io/ingress-gce/pkg/neg/types"
 	"k8s.io/ingress-gce/pkg/network"
@@ -53,6 +54,13 @@ func StartNEGController(
 	providerConfigName := providerConfig.Name
 	logger.V(2).Info("Initializing NEG controller", "providerConfig", providerConfigName)
 
+	// This is the ID for tenant/cluster for which the NEG controller is created.
+	providerConfigID := ""
+	if providerConfig.Spec.PrincipalInfo != nil {
+		providerConfigID = providerConfig.Spec.PrincipalInfo.ID
+		logger.V(2).Info("Initializing NEG controller", "providerConfigID", providerConfigID)
+	}
+
 	// The ProviderConfig-specific stop channel. We close this in StopControllersForProviderConfig.
 	providerConfigStopCh := make(chan struct{})
 
@@ -103,6 +111,7 @@ func StartNEGController(
 		lpConfig,
 		joinedStopCh,
 		logger,
+		providerConfigID,
 	)
 
 	if err != nil {
@@ -261,6 +270,7 @@ func createNEGController(
 	lpConfig labels.PodLabelPropagationConfig,
 	stopCh <-chan struct{},
 	logger klog.Logger,
+	providerConfigID string,
 ) (*neg.Controller, error) {
 
 	// The adapter uses Network SelfLink
@@ -272,6 +282,7 @@ func createNEGController(
 
 	noDefaultBackendServicePort := utils.ServicePort{}
 	var noNodeTopologyInformer cache.SharedIndexInformer
+	negMetrics := metrics.NewNegMetrics(providerConfigID)
 
 	negController, err := neg.NewController(
 		kubeClient,
@@ -290,7 +301,7 @@ func createNEGController(
 		hasSynced,
 		l4Namer,
 		noDefaultBackendServicePort,
-		negtypes.NewAdapterWithRateLimitSpecs(cloud, flags.F.GCERateLimit.Values(), adapter),
+		negtypes.NewAdapterWithRateLimitSpecs(cloud, flags.F.GCERateLimit.Values(), adapter, negMetrics),
 		zoneGetter,
 		clusterNamer,
 		flags.F.ResyncPeriod,
@@ -307,6 +318,7 @@ func createNEGController(
 		flags.F.ReadOnlyMode,
 		stopCh,
 		logger,
+		negMetrics,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create NEG controller: %w", err)

pkg/neg/controller.go

@@ -39,7 +39,7 @@ import (
 	svcnegv1beta1 "k8s.io/ingress-gce/pkg/apis/svcneg/v1beta1"
 	"k8s.io/ingress-gce/pkg/controller/translator"
 	"k8s.io/ingress-gce/pkg/flags"
-	"k8s.io/ingress-gce/pkg/neg/metrics"
+	metrics "k8s.io/ingress-gce/pkg/neg/metrics"
 	"k8s.io/ingress-gce/pkg/neg/metrics/metricscollector"
 	syncMetrics "k8s.io/ingress-gce/pkg/neg/metrics/metricscollector"
 	"k8s.io/ingress-gce/pkg/neg/readiness"
@@ -118,6 +118,9 @@ type Controller struct {
 
 	stopCh <-chan struct{}
 	logger klog.Logger
+
+	// negMetrics is used to collect metrics for NEG
+	negMetrics *metrics.NegMetrics
 }
 
 // NewController returns a network endpoint group controller.
@@ -155,6 +158,7 @@ func NewController(
 	readOnlyMode bool,
 	stopCh <-chan struct{},
 	logger klog.Logger,
+	negMetrics *metrics.NegMetrics,
 ) (*Controller, error) {
 	if svcNegClient == nil {
 		return nil, fmt.Errorf("svcNegClient is nil")
@@ -170,17 +174,18 @@ func NewController(
 	err := scheme.AddToScheme(negScheme)
 	if err != nil {
 		logger.Error(err, "Errored adding default scheme to event recorder")
-		metrics.PublishNegControllerErrorCountMetrics(err, true)
+		negMetrics.PublishNegControllerErrorCountMetrics(err, true)
 	}
 	err = svcnegv1beta1.AddToScheme(negScheme)
 	if err != nil {
 		logger.Error(err, "Errored adding NEG CRD scheme to event recorder")
-		metrics.PublishNegControllerErrorCountMetrics(err, true)
+		negMetrics.PublishNegControllerErrorCountMetrics(err, true)
 	}
 	recorder := eventBroadcaster.NewRecorder(negScheme,
 		apiv1.EventSource{Component: "neg-controller"})
 
-	syncerMetrics := syncMetrics.NewNegMetricsCollector(flags.F.NegMetricsExportInterval, logger)
+	syncerMetrics := syncMetrics.NewNegMetricsCollector(flags.F.NegMetricsExportInterval, logger, negMetrics.ProviderConfigID)
+
 	manager := newSyncerManager(
 		namer,
 		l4Namer,
@@ -199,7 +204,9 @@ func NewController(
 		enableDualStackNEG,
 		numGCWorkers,
 		lpConfig,
-		logger)
+		logger,
+		negMetrics,
+	)
 
 	var reflector readiness.Reflector
 	if enableReadinessReflector {
@@ -213,6 +220,7 @@ func NewController(
 			enableDualStackNEG,
 			flags.F.EnableMultiSubnetCluster && !flags.F.EnableMultiSubnetClusterPhase1,
 			logger,
+			negMetrics,
 		)
 	} else {
 		reflector = &readiness.NoopReflector{}
@@ -255,6 +263,7 @@ func NewController(
 		readOnlyMode:                   readOnlyMode,
 		stopCh:                         stopCh,
 		logger:                         logger,
+		negMetrics:                     negMetrics,
 	}
 	if enableMultiSubnetClusterPhase1 {
 		negController.nodeTopologyQueue = workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "neg_node_topology_queue")
@@ -485,7 +494,7 @@ func (c *Controller) processEndpoint(key string) {
 	namespace, name, err := cache.SplitMetaNamespaceKey(key)
 	if err != nil {
 		c.logger.Error(err, "Failed to split endpoint namespaced key", "key", key)
-		metrics.PublishNegControllerErrorCountMetrics(err, true)
+		c.negMetrics.PublishNegControllerErrorCountMetrics(err, true)
 		return
 	}
 	c.manager.Sync(namespace, name)
@@ -501,7 +510,7 @@ func (c *Controller) serviceWorker() {
 			defer c.serviceQueue.Done(key)
 			err := c.processService(key.(string))
 			c.handleErr(err, key)
-			metrics.PublishNegControllerErrorCountMetrics(err, false)
+			c.negMetrics.PublishNegControllerErrorCountMetrics(err, false)
 		}()
 	}
 }
@@ -862,7 +871,7 @@ func (c *Controller) handleErr(err error, key interface{}) {
 	c.logger.Error(nil, msg)
 	if service, exists, err := c.serviceLister.GetByKey(key.(string)); err != nil {
 		c.logger.Error(err, "Failed to retrieve service from store", "service", key.(string))
-		metrics.PublishNegControllerErrorCountMetrics(err, true)
+		c.negMetrics.PublishNegControllerErrorCountMetrics(err, true)
 	} else if exists {
 		c.recorder.Eventf(service.(*apiv1.Service), apiv1.EventTypeWarning, "ProcessServiceFailed", msg)
 	}
@@ -885,7 +894,7 @@ func (c *Controller) enqueueEndpointSlice(obj interface{}) {
 	key, err := endpointslices.EndpointSlicesServiceKey(endpointSlice)
 	if err != nil {
 		c.logger.Error(err, "Failed to find a service label inside endpoint slice", "endpointSlice", klog.KObj(endpointSlice))
-		metrics.PublishNegControllerErrorCountMetrics(err, true)
+		c.negMetrics.PublishNegControllerErrorCountMetrics(err, true)
 		return
 	}
 	c.logger.V(3).Info("Adding EndpointSlice to endpointQueue for processing", "endpointSlice", key)
@@ -896,7 +905,7 @@ func (c *Controller) enqueueNode(obj interface{}) {
 	key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
 	if err != nil {
 		c.logger.Error(err, "Failed to generate node key")
-		metrics.PublishNegControllerErrorCountMetrics(err, true)
+		c.negMetrics.PublishNegControllerErrorCountMetrics(err, true)
 		return
 	}
 	c.logger.V(3).Info("Adding Node to nodeQueue for processing", "node", key)
@@ -907,7 +916,7 @@ func (c *Controller) enqueueService(obj interface{}) {
 	key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
 	if err != nil {
 		c.logger.Error(err, "Failed to generate service key")
-		metrics.PublishNegControllerErrorCountMetrics(err, true)
+		c.negMetrics.PublishNegControllerErrorCountMetrics(err, true)
 		return
 	}
 	c.logger.V(3).Info("Adding Service to serviceQueue for processing", "service", key)
@@ -931,7 +940,7 @@ func (c *Controller) enqueueNodeTopology(obj interface{}) {
 	key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
 	if err != nil {
 		c.logger.Error(err, "Failed to generate Node Topology key")
-		metrics.PublishNegControllerErrorCountMetrics(err, true)
+		c.negMetrics.PublishNegControllerErrorCountMetrics(err, true)
 		return
 	}
 	c.logger.V(3).Info("Adding NodeTopology to nodeTopologyQueue for processing", "nodeTopology", key)
@@ -941,7 +950,7 @@ func (c *Controller) enqueueNodeTopology(obj interface{}) {
 func (c *Controller) gc() {
 	if err := c.manager.GC(); err != nil {
 		c.logger.Error(err, "NEG controller garbage collection failed")
-		metrics.PublishNegControllerErrorCountMetrics(err, true)
+		c.negMetrics.PublishNegControllerErrorCountMetrics(err, true)
 	}
 }
 

pkg/neg/controller_test.go

@@ -145,7 +145,7 @@ func newTestControllerWithParamsAndContext(kubeClient kubernetes.Interface, test
 		func() bool { return true },
 		testContext.L4Namer,
 		defaultBackend,
-		negtypes.NewAdapter(testContext.Cloud),
+		negtypes.NewAdapter(testContext.Cloud, testContext.NegMetrics),
 		zoneGetter,
 		testContext.NegNamer,
 		testContext.ResyncPeriod,
@@ -163,6 +163,7 @@ func newTestControllerWithParamsAndContext(kubeClient kubernetes.Interface, test
 		readOnlyMode,
 		make(<-chan struct{}),
 		klog.TODO(),
+		testContext.NegMetrics,
 	)
 }