chore: update metrics service to use v2/telemetry/OtelMetrics (#1719)

* chore: use v2/telemetry/OtelMetrics

* chore: remove metrics response

* chore: fix alert list tests

Author: PengyuanZhao

.github/workflows/release.yml

@@ -35,7 +35,7 @@ jobs:
 
       - name: Build Artifacts
         env:
-          HONEYDATASET: ${{ secrets.PROD_HONEYDATASET }}
+          METRIC_DATASET: ${{ secrets.METRIC_DATASET }}
         run: |
           make prepare
           scripts/release.sh build

.github/workflows/windows-integration.yml

@@ -28,7 +28,6 @@ jobs:
           CI_SUBACCOUNT: ${{ secrets.CI_SUBACCOUNT }}
           CI_API_KEY: ${{ secrets.CI_API_KEY }}
           CI_API_SECRET: ${{ secrets.CI_API_SECRET }}
-          HONEYDATASET: ${{ secrets.HONEYDATASET }}
           LW_INT_TEST_AWS_ACC: ${{ secrets.LW_INT_TEST_AWS_ACC }}
         run: |
           Set-ExecutionPolicy Bypass -Scope Process -Force

Makefile

@@ -17,11 +17,10 @@ COVERAGEHTML?=coverage.html
 GOJUNITOUT?=go-junit.xml
 PACKAGENAME?=lacework-cli
 CLINAME?=lacework
-# Honeycomb variables
-HONEYDATASET?=lacework-cli-dev
+METRIC_DATASET?=lacework-cli-dev
 GO_LDFLAGS="-X github.com/lacework/go-sdk/v2/cli/cmd.Version=$(shell cat VERSION) \
             -X github.com/lacework/go-sdk/v2/cli/cmd.GitSHA=$(shell git rev-parse HEAD) \
-            -X github.com/lacework/go-sdk/v2/cli/cmd.HoneyDataset=$(HONEYDATASET) \
+            -X github.com/lacework/go-sdk/v2/cli/cmd.MetricDataset=$(METRIC_DATASET) \
             -X github.com/lacework/go-sdk/v2/cli/cmd.BuildTime=$(shell date +%Y%m%d%H%M%S)"
 GOFLAGS=-mod=vendor
 CGO_ENABLED?=0

api/api.go

@@ -27,7 +27,7 @@ const (
 	// API v2 Endpoints
 	apiTokens = "v2/access/tokens" // Auth
 
-	apiV2HoneyMetrics = "v2/metrics/honeycomb"
+	apiV2OtelMetrics = "v2/telemetry/OtelMetrics?dataset=%s"
 
 	apiV2UserProfile = "v2/UserProfile"
 

api/metrics.go

@@ -19,6 +19,7 @@
 package api
 
 import (
+	"fmt"
 	"os"
 	"runtime"
 )
@@ -30,18 +31,22 @@ type MetricsService struct {
 	client *Client
 }
 
-func (svc *MetricsService) Send(event Honeyvent) (response HoneyEventResponse, err error) {
+func (svc *MetricsService) Send(event MetricEvent) (err error) {
 	if disabled := os.Getenv(DisableTelemetry); disabled != "" {
-		return HoneyEventResponse{Data: []Honeyvent{{TraceID: "Telemetry Disabled"}}}, nil
+		return
 	}
 
 	event.setAccountDetails(*svc.client)
-	err = svc.client.RequestEncoderDecoder("POST", apiV2HoneyMetrics, event, &response)
+	event.SampleRate100 = true
+	event.TelemetrySource = "external"
+	event.TelemetryType = "customer"
+
+	err = svc.client.RequestEncoderDecoder("POST", fmt.Sprintf(apiV2OtelMetrics, event.Dataset), event, nil)
 	return
 }
 
-func NewHoneyvent(version, feature, dataset string) Honeyvent {
-	event := Honeyvent{
+func NewMetricEvent(version, feature, dataset string) MetricEvent {
+	event := MetricEvent{
 		Os:      runtime.GOOS,
 		Arch:    runtime.GOARCH,
 		TraceID: newID(),
@@ -53,7 +58,7 @@ func NewHoneyvent(version, feature, dataset string) Honeyvent {
 	return event
 }
 
-func (h *Honeyvent) setAccountDetails(client Client) {
+func (h *MetricEvent) setAccountDetails(client Client) {
 	if h.Account == "" {
 		h.Account = client.account
 	}
@@ -62,8 +67,8 @@ func (h *Honeyvent) setAccountDetails(client Client) {
 	}
 }
 
-// Honeyvent defines what a Honeycomb event looks like for the Lacework CLI
-type Honeyvent struct {
+// MetricEvent defines what a metric event looks like for the Lacework CLI
+type MetricEvent struct {
 	Version       string      `json:"version"`
 	CfgVersion    int         `json:"config_version"`
 	Os            string      `json:"os"`
@@ -89,15 +94,13 @@ type Honeyvent struct {
 	SpanID    string `json:"trace.span_id,omitempty"`
 	ParentID  string `json:"trace.parent_id,omitempty"`
 	ContextID string `json:"trace.context_id,omitempty"`
-}
 
-type HoneyEventResponse struct {
-	Data    []Honeyvent `json:"data"`
-	Ok      bool        `json:"ok"`
-	Message string      `json:"message"`
+	SampleRate100   bool   `json:"sample_rate_100,omitempty"`
+	TelemetrySource string `json:"telemetry_source,omitempty"`
+	TelemetryType   string `json:"telemetry_type,omitempty"`
 }
 
-func (e *Honeyvent) AddFeatureField(key string, value interface{}) {
+func (e *MetricEvent) AddFeatureField(key string, value interface{}) {
 	if e.FeatureData == nil {
 		e.FeatureData = map[string]interface{}{key: value}
 		return

api/metrics_test.go

@@ -21,8 +21,7 @@ func TestDisableTelemetry(t *testing.T) {
 
 	assert.Nil(t, err)
 
-	res, err := c.V2.Metrics.Send(api.Honeyvent{})
+	err = c.V2.Metrics.Send(api.MetricEvent{})
 
 	assert.Nil(t, err)
-	assert.Equal(t, res.Data[0].TraceID, "Telemetry Disabled")
 }

cli/cmd/cdk.go

@@ -121,7 +121,7 @@ func (c *cliState) Honeyvent(ctx context.Context, in *cdk.HoneyventRequest) (*cd
 	}
 
 	// Send the Honeyvent
-	c.SendHoneyvent()
+	c.SendMetricEvent()
 
 	return &cdk.Reply{}, nil
 }

cli/cmd/cli_state.go

@@ -61,7 +61,7 @@ type cliState struct {
 	LwApi        *api.Client
 	JsonF        *prettyjson.Formatter
 	Log          *zap.SugaredLogger
-	Event        *api.Honeyvent
+	Event        *api.MetricEvent
 	Cache        *diskv.Diskv
 	LwComponents *lwcomponent.State
 
@@ -108,8 +108,8 @@ func NewDefaultState() *cliState {
 		cdkServerPort:  defaultGrpcPort,
 	}
 
-	// initialize honeycomb library and honeyvent
-	c.InitHoneyvent()
+	// initialize MetricEvent
+	c.InitMetricEvent()
 
 	return c
 }
@@ -135,7 +135,7 @@ func (c *cliState) SetProfile(profile string) error {
 // this function verifies parameters and env variables coming from viper
 func (c *cliState) LoadState() error {
 	defer func() {
-		// update global honeyvent with loaded state
+		// update global MetricEvent with loaded state
 		c.Event.Account = c.Account
 		c.Event.Subaccount = c.Subaccount
 		c.Event.Profile = c.Profile

cli/cmd/component.go

@@ -419,7 +419,7 @@ func installComponent(args []string) (err error) {
 
 	cli.Event.Component = componentName
 	cli.Event.Feature = "install_component"
-	defer cli.SendHoneyvent()
+	defer cli.SendMetricEvent()
 
 	catalog, err := LoadCatalog(componentName, false)
 	if err != nil {
@@ -896,7 +896,7 @@ func prototypeRunComponentsInstall(_ *cobra.Command, args []string) (err error)
 
 	cli.Event.Component = componentName
 	cli.Event.Feature = "install_component"
-	defer cli.SendHoneyvent()
+	defer cli.SendMetricEvent()
 
 	cli.StartProgress("Loading components state...")
 	// @afiune maybe move the state to the cache and fetch if it if has expired

cli/cmd/honeyvent.go renamed to cli/cmd/metricevent.go

@@ -25,22 +25,21 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/honeycombio/libhoney-go"
 	"github.com/lacework/go-sdk/v2/api"
 	"github.com/lacework/go-sdk/v2/lwdomain"
 )
 
 var (
-	// HoneyDataset is the dataset in Honeycomb that we send tracing
+	// MetricDataset is the dataset in Elastic that we send tracing
 	// data this variable will be set depending on the environment we
 	// are running on. During development, we send all events and
 	// tracing data to a default dataset.
-	HoneyDataset = "lacework-cli-dev"
+	MetricDataset = "lacework-cli-dev"
 )
 
 const (
 	// DisableTelemetry is an environment variable that can be used to
-	// disable telemetry sent to Honeycomb
+	// disable telemetry sent to Elastic
 	DisableTelemetry = "LW_TELEMETRY_DISABLE"
 
 	// HomebrewInstall is an environment variable that denotes the
@@ -54,13 +53,13 @@ const (
 	// List of Features
 	//
 	// A feature within the Lacework CLI is any functionality that
-	// can't be traced or tracked by the default event sent to Honeycomb,
+	// can't be traced or tracked by the default event sent to Elastic,
 	// it is a behavior that we, Lacework engineers, would like to
 	// trace and understand its usage and adoption.
 	//
-	// By default the Feature field within the Honeyvent is empty,
+	// By default the Feature field within the MetricEvent is empty,
 	// define a new feature below and set it before sending a new
-	// Honeyvent. Additionally, there is a FeatureData field that
+	// MetricEvent. Additionally, there is a FeatureData field that
 	// any feature can use to inject any specific information
 	// related to that feature.
 	//
@@ -69,7 +68,7 @@ const (
 	// ```go
 	// cli.Event.Feature = featPollCtrScan
 	// cli.Event.AddFeatureField("key", "value")
-	// cli.SendHoneyvent()
+	// cli.SendMetricEvent()
 	// ```
 	//
 	// Polling mechanism feature
@@ -91,11 +90,11 @@ const (
 	featMigrateConfigV2 = "migrate_config_v2"
 )
 
-// InitHoneyvent initialize honeycomb library and main Honeyvent, such event
+// InitMetricEvent initialize Elastic library and main MetricEvent, such event
 // could be modified during a command execution to add extra parameters such
 // as error message, feature data, etc.
-func (c *cliState) InitHoneyvent() {
-	c.Event = &api.Honeyvent{
+func (c *cliState) InitMetricEvent() {
+	c.Event = &api.MetricEvent{
 		Os:            runtime.GOOS,
 		Arch:          runtime.GOARCH,
 		Version:       Version,
@@ -106,36 +105,33 @@ func (c *cliState) InitHoneyvent() {
 		CfgVersion:    c.CfgVersion,
 		TraceID:       newID(),
 		InstallMethod: installMethod(),
-		Dataset:       HoneyDataset,
+		Dataset:       MetricDataset,
 	}
 }
 
 // Wait should be called before finishing the execution of any CLI command,
-// it waits for pending workers (a.k.a. honeyvents) to be transmitted
+// it waits for pending workers (a.k.a. MetricEvents) to be transmitted
 func (c *cliState) Wait() {
 	// wait for any missing worker
 	c.workers.Wait()
 
-	// flush any pending calls to Honeycomb
-	libhoney.Close()
-
 	// stop gRPC server gracefully
 	c.Stop()
 }
 
-// SendHoneyvent is used throughout the CLI to send Honeyvents, these events
+// SendMetricEvent is used throughout the CLI to send metric events, these events
 // have tracing data to understand how the commands are being executed, what
 // features are used and the overall command flow. This function sends the
 // events via goroutines so that we don't block the execution of the main process
 //
 // NOTE: the CLI will send at least one event per command execution
-func (c *cliState) SendHoneyvent() {
+func (c *cliState) SendMetricEvent() {
 	if disabled := os.Getenv(DisableTelemetry); disabled != "" {
 		return
 	}
 
 	if c.LwApi == nil {
-		c.Log.Debug("unable to send honeyvent", "error")
+		c.Log.Debug("unable to send MetricEvent", "error")
 		return
 	}
 
@@ -154,8 +150,7 @@ func (c *cliState) SendHoneyvent() {
 
 	// Lacework accounts are NOT case-sensitive but some users configure them
 	// in uppercase and others in lowercase, therefore we will normalize all
-	// account to be lowercase so that we don't see different accounts in
-	// Honeycomb.
+	// account to be lowercase so that we don't see different accounts in metrics.
 	c.Event.Account = strings.ToLower(c.Event.Account)
 
 	// Detect if the account has the full domain, if so, subtract the account
@@ -166,31 +161,30 @@ func (c *cliState) SendHoneyvent() {
 		}
 	}
 
-	c.Log.Debugw("new honeyvent", "dataset", HoneyDataset,
+	c.Log.Debugw("new metric event", "dataset", MetricDataset,
 		"trace_id", c.Event.TraceID,
 		"span_id", c.Event.SpanID,
 		"parent_id", c.Event.ParentID,
 		"context_id", c.Event.ContextID,
 	)
-	honeyvent := libhoney.NewEvent()
-	_ = honeyvent.Add(c.Event)
-	honeycombEvent := *c.Event
-	honeycombEvent.Dataset = c.Event.Dataset
+
+	event := *c.Event
+	event.Dataset = c.Event.Dataset
 
 	c.workers.Add(1)
-	go func(wg *sync.WaitGroup, event *libhoney.Event, honeycombEvent api.Honeyvent) {
+	go func(wg *sync.WaitGroup, event api.MetricEvent) {
 		defer wg.Done()
 
-		c.Log.Debugw("sending honeyvent", "dataset", HoneyDataset)
+		c.Log.Debugw("sending MetricEvent", "dataset", MetricDataset)
 
-		_, err := c.LwApi.V2.Metrics.Send(honeycombEvent)
+		err := c.LwApi.V2.Metrics.Send(event)
 		if err != nil {
-			c.Log.Debugw("unable to send honeyvent", "error", err)
+			c.Log.Debugw("unable to send MetricEvent", "error", err)
 		}
 
-	}(&c.workers, honeyvent, honeycombEvent)
+	}(&c.workers, event)
 
-	// after adding a worker to submit a honeyvent, we remove
+	// after adding a worker to submit a metric event, we remove
 	// all temporal fields such as feature, feature.data, error
 	c.Event.DurationMs = 0
 	c.Event.Error = ""