Fixed metrics collector aggregation issue for NEG metrics in MT

Author: priyapande

cmd/glbc/main.go

@@ -59,18 +59,19 @@ import (
 	"k8s.io/ingress-gce/pkg/utils"
 	"k8s.io/klog/v2"
 
-	ingctx "k8s.io/ingress-gce/pkg/context"
-	"k8s.io/ingress-gce/pkg/controller"
-	"k8s.io/ingress-gce/pkg/neg"
-	"k8s.io/ingress-gce/pkg/neg/syncers/labels"
-	negtypes "k8s.io/ingress-gce/pkg/neg/types"
-
 	"k8s.io/ingress-gce/cmd/glbc/app"
 	"k8s.io/ingress-gce/pkg/backendconfig"
+	ingctx "k8s.io/ingress-gce/pkg/context"
+	"k8s.io/ingress-gce/pkg/controller"
 	"k8s.io/ingress-gce/pkg/crd"
 	"k8s.io/ingress-gce/pkg/firewalls"
 	"k8s.io/ingress-gce/pkg/flags"
 	_ "k8s.io/ingress-gce/pkg/klog"
+	"k8s.io/ingress-gce/pkg/neg"
+	"k8s.io/ingress-gce/pkg/neg/metrics"
+	syncMetrics "k8s.io/ingress-gce/pkg/neg/metrics/metricscollector"
+	"k8s.io/ingress-gce/pkg/neg/syncers/labels"
+	negtypes "k8s.io/ingress-gce/pkg/neg/types"
 	"k8s.io/ingress-gce/pkg/utils/zonegetter"
 	"k8s.io/ingress-gce/pkg/version"
 )
@@ -266,6 +267,8 @@ func main() {
 				nodeTopologyFactory = informernodetopology.NewSharedInformerFactory(nodeTopologyClient, flags.F.ResyncPeriod)
 			}
 			ctx := context.Background()
+			syncerMetrics := syncMetrics.NewNegMetricsCollector(flags.F.NegMetricsExportInterval, rootLogger)
+
 			if flags.F.LeaderElection.LeaderElect {
 				err := multiprojectstart.StartWithLeaderElection(
 					ctx,
@@ -284,6 +287,7 @@ func main() {
 					gceCreator,
 					namer,
 					stopCh,
+					syncerMetrics,
 				)
 				if err != nil {
 					rootLogger.Error(err, "Failed to start multi-project syncer with leader election")
@@ -304,6 +308,7 @@ func main() {
 					gceCreator,
 					namer,
 					stopCh,
+					syncerMetrics,
 				)
 			}
 		}, rOption.wg)
@@ -711,6 +716,13 @@ func createNEGController(ctx *ingctx.ControllerContext, systemHealth *systemheal
 		adapter = ctx.Cloud
 	}
 
+	// register NEG prometheus metrics
+	metrics.RegisterMetrics()
+	syncMetrics.RegisterMetrics()
+
+	negMetrics := metrics.NewNegMetrics()
+	syncerMetrics := syncMetrics.NewNegMetricsCollector(flags.F.NegMetricsExportInterval, logger)
+
 	// TODO: Refactor NEG to use cloud mocks so ctx.Cloud can be referenced within NewController.
 	negController, err := neg.NewController(
 		ctx.KubeClient,
@@ -729,7 +741,7 @@ func createNEGController(ctx *ingctx.ControllerContext, systemHealth *systemheal
 		ctx.HasSynced,
 		ctx.L4Namer,
 		ctx.DefaultBackendSvcPort,
-		negtypes.NewAdapterWithRateLimitSpecs(ctx.Cloud, flags.F.GCERateLimit.Values(), adapter),
+		negtypes.NewAdapterWithRateLimitSpecs(ctx.Cloud, flags.F.GCERateLimit.Values(), adapter, negMetrics),
 		zoneGetter,
 		ctx.ClusterNamer,
 		flags.F.ResyncPeriod,
@@ -747,6 +759,8 @@ func createNEGController(ctx *ingctx.ControllerContext, systemHealth *systemheal
 		flags.F.EnableNEGsForIngress,
 		stopCh,
 		logger,
+		negMetrics,
+		syncerMetrics,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create NEG controller: %w", err)

docs/deploy/local/README.md

@@ -32,21 +32,14 @@ from the root of this repo:
 $ hack/setup-local.sh <cluster-name>
 ```
 
-## Setup GCE permissions
+## Authorize gcloud and kubectl
 
-When running locally, the Ingress-GCE controller looks on the local machine
-for credentials to create GCE networking resources. Specifically it looks for a
-json file specified at the GOOGLE_APPLICATION_CREDENTIALS variable. Given this,
-it is most desirable to follow these steps:
-
-1. Create a Service Account in GCP and give the account Compute Admin permissions
-
-2. Create a key for the Service Account and download it
-
-Then run the following:
+Once the cluster is ready prepare authorization to it.
+You need to authorize both gcloud and kubectl.
 
 ```console
-export GOOGLE_APPLICATION_CREDENTIALS=/path/to/key/file
+$ gcloud auth application-default login
+$ gcloud container clusters get-credentials CLUSTER_NAME --region CLUSTER_LOCATION
 ```
 
 ## Run the controller
@@ -62,6 +55,15 @@ binary in a container and place it in `bin/amd64`.
 make build
 ```
 
+NOTE -
+If you get build errors with Docker getting a permission denied while pulling the base image run the following commands:
+
+```console
+sudo addgroup --system docker
+sudo usermod -aG docker $USER
+newgrp docker
+```
+
 For Mac OS users or to build the binary locally and output it in the
 `bin/amd64` directory run:
 

pkg/controller/controller.go

@@ -48,6 +48,7 @@ import (
 	"k8s.io/ingress-gce/pkg/loadbalancers"
 	"k8s.io/ingress-gce/pkg/loadbalancers/features"
 	"k8s.io/ingress-gce/pkg/metrics"
+	negmetrics "k8s.io/ingress-gce/pkg/neg/metrics"
 	negtypes "k8s.io/ingress-gce/pkg/neg/types"
 	ingsync "k8s.io/ingress-gce/pkg/sync"
 	"k8s.io/ingress-gce/pkg/translator"
@@ -139,7 +140,7 @@ func NewLoadBalancerController(
 		instancePool:                   ctx.InstancePool,
 		l7Pool:                         loadbalancers.NewLoadBalancerPool(ctx.Cloud, ctx.ClusterNamer, ctx, namer.NewFrontendNamerFactory(ctx.ClusterNamer, ctx.KubeSystemUID, logger), logger),
 		backendSyncer:                  backends.NewBackendSyncer(backendPool, healthChecker, ctx.Cloud, ctx.Translator),
-		negLinker:                      backends.NewNEGLinker(backendPool, negtypes.NewAdapter(ctx.Cloud), ctx.Cloud, ctx.SvcNegInformer.GetIndexer(), logger),
+		negLinker:                      backends.NewNEGLinker(backendPool, negtypes.NewAdapter(ctx.Cloud, negmetrics.NewNegMetrics()), ctx.Cloud, ctx.SvcNegInformer.GetIndexer(), logger),
 		igLinker:                       backends.NewInstanceGroupLinker(ctx.InstancePool, backendPool, logger),
 		metrics:                        ctx.ControllerMetrics,
 		ZoneGetter:                     ctx.ZoneGetter,

pkg/l4lb/l4controller.go

@@ -38,6 +38,7 @@ import (
 	"k8s.io/ingress-gce/pkg/forwardingrules"
 	"k8s.io/ingress-gce/pkg/l4lb/metrics"
 	"k8s.io/ingress-gce/pkg/loadbalancers"
+	negmetrics "k8s.io/ingress-gce/pkg/neg/metrics"
 	negtypes "k8s.io/ingress-gce/pkg/neg/types"
 	"k8s.io/ingress-gce/pkg/network"
 	"k8s.io/ingress-gce/pkg/utils"
@@ -106,7 +107,7 @@ func NewILBController(ctx *context.ControllerContext, stopCh <-chan struct{}, lo
 		hasSynced:       ctx.HasSynced,
 	}
 	l4c.backendPool = backends.NewPool(ctx.Cloud, l4c.namer)
-	l4c.NegLinker = backends.NewNEGLinker(l4c.backendPool, negtypes.NewAdapter(ctx.Cloud), ctx.Cloud, ctx.SvcNegInformer.GetIndexer(), logger)
+	l4c.NegLinker = backends.NewNEGLinker(l4c.backendPool, negtypes.NewAdapter(ctx.Cloud, negmetrics.NewNegMetrics()), ctx.Cloud, ctx.SvcNegInformer.GetIndexer(), logger)
 
 	l4c.svcQueue = utils.NewPeriodicTaskQueueWithMultipleWorkers("l4", "services", l4c.numWorkers, l4c.syncWrapper, logger)
 

pkg/l4lb/l4netlbcontroller.go

@@ -39,6 +39,7 @@ import (
 	"k8s.io/ingress-gce/pkg/instancegroups"
 	"k8s.io/ingress-gce/pkg/l4lb/metrics"
 	"k8s.io/ingress-gce/pkg/loadbalancers"
+	negmetrics "k8s.io/ingress-gce/pkg/neg/metrics"
 	negtypes "k8s.io/ingress-gce/pkg/neg/types"
 	"k8s.io/ingress-gce/pkg/network"
 	"k8s.io/ingress-gce/pkg/utils"
@@ -138,7 +139,7 @@ func NewL4NetLBController(
 		adapter = ctx.Cloud
 	}
 	l4netLBc.networkResolver = network.NewNetworksResolver(networkLister, gkeNetworkParamSetLister, adapter, ctx.EnableMultinetworking, logger)
-	l4netLBc.negLinker = backends.NewNEGLinker(l4netLBc.backendPool, negtypes.NewAdapter(ctx.Cloud), ctx.Cloud, ctx.SvcNegInformer.GetIndexer(), logger)
+	l4netLBc.negLinker = backends.NewNEGLinker(l4netLBc.backendPool, negtypes.NewAdapter(ctx.Cloud, negmetrics.NewNegMetrics()), ctx.Cloud, ctx.SvcNegInformer.GetIndexer(), logger)
 	l4netLBc.svcQueue = utils.NewPeriodicTaskQueueWithMultipleWorkers("l4netLB", "services", ctx.NumL4NetLBWorkers, l4netLBc.syncWrapper, logger)
 
 	ctx.ServiceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{

pkg/multiproject/manager/manager.go

@@ -15,6 +15,7 @@ import (
 	"k8s.io/ingress-gce/pkg/multiproject/finalizer"
 	"k8s.io/ingress-gce/pkg/multiproject/gce"
 	"k8s.io/ingress-gce/pkg/multiproject/neg"
+	syncMetrics "k8s.io/ingress-gce/pkg/neg/metrics/metricscollector"
 	"k8s.io/ingress-gce/pkg/neg/syncers/labels"
 	providerconfigclient "k8s.io/ingress-gce/pkg/providerconfig/client/clientset/versioned"
 	svcnegclient "k8s.io/ingress-gce/pkg/svcneg/client/clientset/versioned"
@@ -44,6 +45,7 @@ type ProviderConfigControllersManager struct {
 	lpConfig             labels.PodLabelPropagationConfig
 	gceCreator           gce.GCECreator
 	globalStopCh         <-chan struct{}
+	syncerMetrics        *syncMetrics.SyncerMetrics
 }
 
 type ControllerSet struct {
@@ -66,6 +68,7 @@ func NewProviderConfigControllerManager(
 	gceCreator gce.GCECreator,
 	globalStopCh <-chan struct{},
 	logger klog.Logger,
+	syncerMetrics *syncMetrics.SyncerMetrics,
 ) *ProviderConfigControllersManager {
 	return &ProviderConfigControllersManager{
 		controllers:          make(map[string]*ControllerSet),
@@ -84,6 +87,7 @@ func NewProviderConfigControllerManager(
 		lpConfig:             lpConfig,
 		gceCreator:           gceCreator,
 		globalStopCh:         globalStopCh,
+		syncerMetrics:        syncerMetrics,
 	}
 }
 
@@ -133,6 +137,7 @@ func (pccm *ProviderConfigControllersManager) StartControllersForProviderConfig(
 		pccm.globalStopCh,
 		logger,
 		pc,
+		pccm.syncerMetrics,
 	)
 	if err != nil {
 		cleanupErr := finalizer.DeleteProviderConfigNEGCleanupFinalizer(pc, pccm.providerConfigClient, logger)

pkg/multiproject/neg/neg.go

@@ -16,6 +16,8 @@ import (
 	"k8s.io/ingress-gce/pkg/flags"
 	"k8s.io/ingress-gce/pkg/multiproject/filteredinformer"
 	"k8s.io/ingress-gce/pkg/neg"
+	"k8s.io/ingress-gce/pkg/neg/metrics"
+	syncMetrics "k8s.io/ingress-gce/pkg/neg/metrics/metricscollector"
 	"k8s.io/ingress-gce/pkg/neg/syncers/labels"
 	negtypes "k8s.io/ingress-gce/pkg/neg/types"
 	"k8s.io/ingress-gce/pkg/network"
@@ -28,6 +30,12 @@ import (
 	"k8s.io/klog/v2"
 )
 
+func init() {
+	// register prometheus metrics
+	metrics.RegisterMetrics()
+	syncMetrics.RegisterMetrics()
+}
+
 // StartNEGController creates and runs a NEG controller for the specified ProviderConfig.
 // The returned channel is closed by StopControllersForProviderConfig to signal a shutdown
 // specific to this ProviderConfig's controller.
@@ -60,10 +68,18 @@ func StartNEGController(
 	globalStopCh <-chan struct{},
 	logger klog.Logger,
 	providerConfig *providerconfig.ProviderConfig,
+	syncerMetrics *syncMetrics.SyncerMetrics,
 ) (chan<- struct{}, error) {
 	providerConfigName := providerConfig.Name
 	logger.V(2).Info("Initializing NEG controller", "providerConfig", providerConfigName)
 
+	// This is the ID for tenant/cluster for which the NEG controller is created.
+	providerConfigID := ""
+	if providerConfig.Spec.PrincipalInfo != nil {
+		providerConfigID = providerConfig.Spec.PrincipalInfo.ID
+		logger.V(2).Info("Initializing NEG controller", "providerConfigID", providerConfigID)
+	}
+
 	// The ProviderConfig-specific stop channel. We close this in StopControllersForProviderConfig.
 	providerConfigStopCh := make(chan struct{})
 
@@ -114,6 +130,8 @@ func StartNEGController(
 		lpConfig,
 		joinedStopCh,
 		logger,
+		providerConfigID,
+		syncerMetrics,
 	)
 
 	if err != nil {
@@ -272,6 +290,8 @@ func createNEGController(
 	lpConfig labels.PodLabelPropagationConfig,
 	stopCh <-chan struct{},
 	logger klog.Logger,
+	providerConfigID string,
+	syncerMetrics *syncMetrics.SyncerMetrics,
 ) (*neg.Controller, error) {
 
 	// The adapter uses Network SelfLink
@@ -283,7 +303,7 @@ func createNEGController(
 
 	noDefaultBackendServicePort := utils.ServicePort{}
 	var noNodeTopologyInformer cache.SharedIndexInformer
-
+	negMetrics := metrics.NewNegMetrics()
 	negController, err := neg.NewController(
 		kubeClient,
 		svcNegClient,
@@ -301,7 +321,7 @@ func createNEGController(
 		hasSynced,
 		l4Namer,
 		noDefaultBackendServicePort,
-		negtypes.NewAdapterWithRateLimitSpecs(cloud, flags.F.GCERateLimit.Values(), adapter),
+		negtypes.NewAdapterWithRateLimitSpecs(cloud, flags.F.GCERateLimit.Values(), adapter, negMetrics),
 		zoneGetter,
 		clusterNamer,
 		flags.F.ResyncPeriod,
@@ -319,6 +339,8 @@ func createNEGController(
 		flags.F.EnableNEGsForIngress,
 		stopCh,
 		logger,
+		negMetrics,
+		syncerMetrics,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create NEG controller: %w", err)

pkg/multiproject/start/start.go

@@ -19,10 +19,13 @@ import (
 	pccontroller "k8s.io/ingress-gce/pkg/multiproject/controller"
 	"k8s.io/ingress-gce/pkg/multiproject/gce"
 	"k8s.io/ingress-gce/pkg/multiproject/manager"
+	"k8s.io/ingress-gce/pkg/neg/metrics"
+	syncMetrics "k8s.io/ingress-gce/pkg/neg/metrics/metricscollector"
 	"k8s.io/ingress-gce/pkg/neg/syncers/labels"
 	providerconfigclient "k8s.io/ingress-gce/pkg/providerconfig/client/clientset/versioned"
 	providerconfiginformers "k8s.io/ingress-gce/pkg/providerconfig/client/informers/externalversions"
 	"k8s.io/ingress-gce/pkg/recorders"
+
 	svcnegclient "k8s.io/ingress-gce/pkg/svcneg/client/clientset/versioned"
 	informersvcneg "k8s.io/ingress-gce/pkg/svcneg/client/informers/externalversions"
 	"k8s.io/ingress-gce/pkg/utils/namer"
@@ -31,6 +34,12 @@ import (
 
 const multiProjectLeaderElectionLockName = "ingress-gce-multi-project-lock"
 
+func init() {
+	// register prometheus metrics
+	metrics.RegisterMetrics()
+	syncMetrics.RegisterMetrics()
+}
+
 // StartWithLeaderElection starts the ProviderConfig controller with leader election.
 func StartWithLeaderElection(
 	parentCtx context.Context,
@@ -49,12 +58,13 @@ func StartWithLeaderElection(
 	gceCreator gce.GCECreator,
 	rootNamer *namer.Namer,
 	stopCh <-chan struct{},
+	syncerMetrics *syncMetrics.SyncerMetrics,
 ) error {
 	logger.V(1).Info("Starting multi-project controller with leader election", "host", hostname)
 
 	recordersManager := recorders.NewManager(eventRecorderKubeClient, logger)
 
-	leConfig, err := makeLeaderElectionConfig(leaderElectKubeClient, hostname, recordersManager, logger, kubeClient, svcNegClient, kubeSystemUID, eventRecorderKubeClient, providerConfigClient, informersFactory, svcNegFactory, networkFactory, nodeTopologyFactory, gceCreator, rootNamer)
+	leConfig, err := makeLeaderElectionConfig(leaderElectKubeClient, hostname, recordersManager, logger, kubeClient, svcNegClient, kubeSystemUID, eventRecorderKubeClient, providerConfigClient, informersFactory, svcNegFactory, networkFactory, nodeTopologyFactory, gceCreator, rootNamer, syncerMetrics)
 	if err != nil {
 		return err
 	}
@@ -88,6 +98,7 @@ func makeLeaderElectionConfig(
 	nodeTopologyFactory informernodetopology.SharedInformerFactory,
 	gceCreator gce.GCECreator,
 	rootNamer *namer.Namer,
+	syncerMetrics *syncMetrics.SyncerMetrics,
 ) (*leaderelection.LeaderElectionConfig, error) {
 	recorder := recordersManager.Recorder(flags.F.LeaderElection.LockObjectNamespace)
 	// add a uniquifier so that two processes on the same host don't accidentally both become active
@@ -116,7 +127,7 @@ func makeLeaderElectionConfig(
 		Callbacks: leaderelection.LeaderCallbacks{
 			OnStartedLeading: func(ctx context.Context) {
 				logger.Info("Became leader, starting multi-project controller")
-				Start(logger, kubeClient, svcNegClient, kubeSystemUID, eventRecorderKubeClient, providerConfigClient, informersFactory, svcNegFactory, networkFactory, nodeTopologyFactory, gceCreator, rootNamer, ctx.Done())
+				Start(logger, kubeClient, svcNegClient, kubeSystemUID, eventRecorderKubeClient, providerConfigClient, informersFactory, svcNegFactory, networkFactory, nodeTopologyFactory, gceCreator, rootNamer, ctx.Done(), syncerMetrics)
 			},
 			OnStoppedLeading: func() {
 				logger.Info("Stop running multi-project leader election")
@@ -143,6 +154,7 @@ func Start(
 	gceCreator gce.GCECreator,
 	rootNamer *namer.Namer,
 	stopCh <-chan struct{},
+	syncerMetrics *syncMetrics.SyncerMetrics,
 ) {
 	logger.V(1).Info("Starting ProviderConfig controller")
 	lpConfig := labels.PodLabelPropagationConfig{}
@@ -173,6 +185,7 @@ func Start(
 		gceCreator,
 		stopCh,
 		logger,
+		syncerMetrics,
 	)
 	logger.V(1).Info("Initialized ProviderConfig controller manager")