From 1dcbe0adcb17a217c39949c3c7935acc81d29bff Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 27 Jul 2023 16:26:06 +0200 Subject: [PATCH 01/62] Draft for serverless --- internal/stack/providers.go | 8 +- internal/stack/serverless.go | 244 +++++++++++++++++++++++++++ internal/stack/serverless/client.go | 201 ++++++++++++++++++++++ internal/stack/serverless/project.go | 75 ++++++++ 4 files changed, 526 insertions(+), 2 deletions(-) create mode 100644 internal/stack/serverless.go create mode 100644 internal/stack/serverless/client.go create mode 100644 internal/stack/serverless/project.go diff --git a/internal/stack/providers.go b/internal/stack/providers.go index e782424b35..730ad7cedb 100644 --- a/internal/stack/providers.go +++ b/internal/stack/providers.go @@ -12,13 +12,15 @@ import ( ) const ( - ProviderCompose = "compose" + ProviderCompose = "compose" + ProviderServerless = "serverless" ) var ( DefaultProvider = ProviderCompose SupportedProviders = []string{ ProviderCompose, + ProviderServerless, } ) @@ -50,8 +52,10 @@ type Provider interface { // BuildProvider returns the provider for the given name. func BuildProvider(name string, profile *profile.Profile) (Provider, error) { switch name { - case "compose": + case ProviderCompose: return &composeProvider{}, nil + case ProviderServerless: + return newServerlessProvider(profile) } return nil, fmt.Errorf("unknown provider %q, supported providers: %s", name, strings.Join(SupportedProviders, ", ")) } diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go new file mode 100644 index 0000000000..87f4d4db66 --- /dev/null +++ b/internal/stack/serverless.go @@ -0,0 +1,244 @@ +package stack + +import ( + "errors" + "fmt" + + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/profile" + "github.com/elastic/elastic-package/internal/stack/serverless" +) + +const ( + paramServerlessProjectID = "serverless_project_id" + paramServerlessProjectType = "serverless_project_type" + + configRegion = "stack.serverless.region" + configProjectType = "stack.serverless.type" + + defaultRegion = "aws-eu-west-1" + defaultProjectType = "observability" +) + +var ( + errProjectNotExist = errors.New("project does not exist") +) + +type serverlessProvider struct { + profile *profile.Profile + client *serverless.Client +} + +type projectSettings struct { + Name string + Region string + Type string + + StackVersion string +} + +func (sp *serverlessProvider) createProject(settings projectSettings, options Options) (Config, error) { + project, err := sp.client.CreateProject(settings.Name, settings.Region, settings.Type) + if err != nil { + return Config{}, fmt.Errorf("failed to create %s project %s in %s: %w", settings.Type, settings.Name, settings.Region, err) + } + + var config Config + config.Provider = ProviderServerless + config.Parameters = map[string]string{ + paramServerlessProjectID: project.ID, + paramServerlessProjectType: project.Type, + } + + config.ElasticsearchHost = project.Endpoints.Elasticsearch + config.KibanaHost = project.Endpoints.Kibana + + printUserConfig(options.Printer, config) + + err = storeConfig(sp.profile, config) + if err != nil { + return Config{}, fmt.Errorf("failed to store config: %w", err) + } + + return config, nil +} + +func (sp *serverlessProvider) deleteProject(project *serverless.Project, options Options) error { + return sp.client.DeleteProject(project) +} + +func (sp *serverlessProvider) currentProject(config Config) (*serverless.Project, error) { + projectID, found := config.Parameters[paramServerlessProjectID] + if !found { + return nil, errProjectNotExist + } + + projectType, found := config.Parameters[paramServerlessProjectType] + if !found { + return nil, errProjectNotExist + } + + project, err := sp.client.GetProject(projectType, projectID) + if err == serverless.ErrProjectNotExist { + return nil, errProjectNotExist + } + if err != nil { + return nil, fmt.Errorf("couldn't check project health: %w", err) + } + + return project, nil +} + +func getProjectSettings(options Options) (projectSettings, error) { + s := projectSettings{ + Name: createProjectName(options), + Type: options.Profile.Config(configProjectType, defaultProjectType), + Region: options.Profile.Config(configRegion, defaultRegion), + StackVersion: options.StackVersion, + } + + return s, nil +} + +func createProjectName(options Options) string { + return fmt.Sprintf("elastic-package-test-%s", options.Profile.ProfileName) +} + +func newServerlessProvider(profile *profile.Profile) (*serverlessProvider, error) { + client, err := serverless.NewClient() + if err != nil { + return nil, fmt.Errorf("can't create serverless provider: %w", err) + } + + return &serverlessProvider{profile, client}, nil +} + +func (sp *serverlessProvider) BootUp(options Options) error { + logger.Warn("Elastic Serverless provider is in technical preview") + + config, err := LoadConfig(sp.profile) + if err != nil { + return fmt.Errorf("failed to load configuration: %w", err) + } + + settings, err := getProjectSettings(options) + if err != nil { + return err + } + + var project *serverless.Project + + project, err = sp.currentProject(config) + switch err { + default: + return err + case errProjectNotExist: + logger.Infof("Creating project %q", settings.Name) + config, err = sp.createProject(settings, options) + if err != nil { + return fmt.Errorf("failed to create deployment: %w", err) + } + + // logger.Infof("Creating agent policy") + // err = sp.createAgentPolicy(config, options.StackVersion) + // if err != nil { + // return fmt.Errorf("failed to create agent policy: %w", err) + // } + + // logger.Infof("Replacing GeoIP databases") + // err = cp.replaceGeoIPDatabases(config, options, settings.TemplateID, settings.Region, payload.Resources.Elasticsearch[0].Plan.ClusterTopology) + // if err != nil { + // return fmt.Errorf("failed to replace GeoIP databases: %w", err) + // } + case nil: + logger.Debugf("Project existed: %s", project.Name) + printUserConfig(options.Printer, config) + logger.Infof("Updating project %s", project.Name) + // err = sp.updateDeployment(project, settings) + // if err != nil { + // return fmt.Errorf("failed to update deployment: %w", err) + // } + } + + // logger.Infof("Starting local agent") + // err = sp.startLocalAgent(options, config) + // if err != nil { + // return fmt.Errorf("failed to start local agent: %w", err) + // } + + return nil +} + +// func (sp *serverlessProvider) startLocalAgent(options Options, config Config) error { +// err := applyCloudResources(sp.profile, options.StackVersion, config) +// if err != nil { +// return fmt.Errorf("could not initialize compose files for local agent: %w", err) +// } +// +// project, err := sp.localAgentComposeProject() +// if err != nil { +// return fmt.Errorf("could not initialize local agent compose project") +// } +// +// err = project.Build(compose.CommandOptions{}) +// if err != nil { +// return fmt.Errorf("failed to build images for local agent: %w", err) +// } +// +// err = project.Up(compose.CommandOptions{ExtraArgs: []string{"-d"}}) +// if err != nil { +// return fmt.Errorf("failed to start local agent: %w", err) +// } +// +// return nil +// } + +func (sp *serverlessProvider) TearDown(options Options) error { + config, err := LoadConfig(sp.profile) + if err != nil { + return fmt.Errorf("failed to load configuration: %w", err) + } + + // err = cp.destroyLocalAgent() + // if err != nil { + // return fmt.Errorf("failed to destroy local agent: %w", err) + // } + + project, err := sp.currentProject(config) + if err != nil { + return fmt.Errorf("failed to find current project: %w", err) + } + + logger.Debugf("Deleting project %q", project.ID) + + err = sp.deleteProject(project, options) + if err != nil { + return fmt.Errorf("failed to delete project: %w", err) + } + + // logger.Debugf("Deleting GeoIP bundle.") + // err = cp.deleteGeoIPExtension() + // if err != nil { + // return fmt.Errorf("failed to delete GeoIP extension: %w", err) + // } + + err = storeConfig(sp.profile, Config{}) + if err != nil { + return fmt.Errorf("failed to store config: %w", err) + } + + return nil +} + +func (sp *serverlessProvider) Update(options Options) error { + return fmt.Errorf("not implemented") +} + +func (sp *serverlessProvider) Dump(options DumpOptions) (string, error) { + return "", fmt.Errorf("not implemented") +} + +func (sp *serverlessProvider) Status(options Options) ([]ServiceStatus, error) { + logger.Warn("Elastic Serverless provider is in technical preview") + return Status(options) +} diff --git a/internal/stack/serverless/client.go b/internal/stack/serverless/client.go new file mode 100644 index 0000000000..87fa903f85 --- /dev/null +++ b/internal/stack/serverless/client.go @@ -0,0 +1,201 @@ +package serverless + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + + "github.com/elastic/elastic-package/internal/environment" + "github.com/elastic/elastic-package/internal/logger" +) + +type Client struct { + host string + apiKey string +} + +// ClientOption is functional option modifying Serverless API client. +type ClientOption func(*Client) + +var ( + ServerlessApiKeyEnvironmentVariable = "SERVERLESS_API_KEY" + ServerlessHostvironmentVariable = "SERVERLESS_HOST" + + ErrProjectNotExist = errors.New("project does not exist") +) + +func NewClient(opts ...ClientOption) (*Client, error) { + hostEnvName := environment.WithElasticPackagePrefix(ServerlessHostvironmentVariable) + host := os.Getenv(hostEnvName) + if host == "" { + return nil, fmt.Errorf("unable to obtain value from %s environment variable", hostEnvName) + } + apiKeyEnvName := environment.WithElasticPackagePrefix(ServerlessApiKeyEnvironmentVariable) + apiKey := os.Getenv(apiKeyEnvName) + if apiKey == "" { + return nil, fmt.Errorf("unable to obtain value from %s environment variable", apiKeyEnvName) + } + c := &Client{ + host: host, + apiKey: apiKey, + } + for _, opt := range opts { + opt(c) + } + + return c, nil +} + +// Address option sets the host to use to connect to Kibana. +func WithAddress(address string) ClientOption { + return func(c *Client) { + c.host = address + } +} + +// Address option sets the host to use to connect to Kibana. +func WithApiKey(apiKey string) ClientOption { + return func(c *Client) { + c.apiKey = apiKey + } +} + +func (c *Client) get(ctx context.Context, resourcePath string) (int, []byte, error) { + return c.sendRequest(ctx, http.MethodGet, resourcePath, nil) +} + +func (c *Client) post(ctx context.Context, resourcePath string, body []byte) (int, []byte, error) { + return c.sendRequest(ctx, http.MethodPost, resourcePath, body) +} + +func (c *Client) delete(ctx context.Context, resourcePath string) (int, []byte, error) { + return c.sendRequest(ctx, http.MethodDelete, resourcePath, nil) +} + +func (c *Client) sendRequest(ctx context.Context, method, resourcePath string, body []byte) (int, []byte, error) { + request, err := c.newRequest(ctx, method, resourcePath, bytes.NewReader(body)) + if err != nil { + return 0, nil, err + } + + return c.doRequest(request) +} + +func (c *Client) newRequest(ctx context.Context, method, resourcePath string, reqBody io.Reader) (*http.Request, error) { + base, err := url.Parse(c.host) + if err != nil { + return nil, fmt.Errorf("could not create base URL from host: %v: %w", c.host, err) + } + + rel, err := url.Parse(resourcePath) + if err != nil { + return nil, fmt.Errorf("could not create relative URL from resource path: %v: %w", resourcePath, err) + } + + u := base.JoinPath(rel.EscapedPath()) + u.RawQuery = rel.RawQuery + + logger.Debugf("%s %s", method, u) + + req, err := http.NewRequestWithContext(ctx, method, u.String(), reqBody) + if err != nil { + return nil, fmt.Errorf("could not create %v request to Kibana API resource: %s: %w", method, resourcePath, err) + } + + req.Header.Add("content-type", "application/json") + req.Header.Add("Authorization", fmt.Sprintf("ApiKey %s", c.apiKey)) + + return req, nil +} + +func (c *Client) doRequest(request *http.Request) (int, []byte, error) { + client := http.Client{} + + resp, err := client.Do(request) + if err != nil { + return 0, nil, fmt.Errorf("could not send request to Kibana API: %w", err) + } + + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return resp.StatusCode, nil, fmt.Errorf("could not read response body: %w", err) + } + + return resp.StatusCode, body, nil +} + +func (c *Client) CreateProject(name, region, project string) (*Project, error) { + ReqBody := struct { + Name string `json:"name"` + RegionID string `json:"region_id"` + }{ + Name: name, + RegionID: region, + } + p, err := json.Marshal(ReqBody) + if err != nil { + return nil, err + } + ctx := context.Background() + resourcePath := fmt.Sprintf("%s/api/v1/serverless/projects/%s", c.host, project) + statusCode, respBody, err := c.post(ctx, resourcePath, p) + + if err != nil { + return nil, fmt.Errorf("error creating project: %w", err) + } + + if statusCode != http.StatusCreated { + return nil, fmt.Errorf("unexpected status code %d, body: %s", statusCode, string(respBody)) + } + + serverlessProject := &Project{url: c.host, apiKey: c.apiKey} + err = json.Unmarshal(respBody, &serverlessProject) + + bytes, _ := json.MarshalIndent(&serverlessProject, "", " ") + fmt.Printf("Project:\n%s", string(bytes)) + + return serverlessProject, err +} + +func (c *Client) DeleteProject(project *Project) error { + ctx := context.Background() + resourcePath := fmt.Sprintf("%s/api/v1/serverless/projects/%s/%s", c.host, project.Type, project.ID) + statusCode, _, err := c.delete(ctx, resourcePath) + if err != nil { + return fmt.Errorf("error deleting project: %w", err) + } + + if statusCode != http.StatusOK { + return fmt.Errorf("unexpected status code %ds", statusCode) + } + + return nil +} + +func (c *Client) GetProject(projectType, projectID string) (*Project, error) { + ctx := context.Background() + resourcePath := fmt.Sprintf("%s/api/v1/serverless/projects/%s/%s", c.host, projectType, projectID) + statusCode, respBody, err := c.get(ctx, resourcePath) + if err != nil { + return nil, fmt.Errorf("error deleting project: %w", err) + } + + if statusCode == http.StatusNotFound { + return nil, ErrProjectNotExist + } + + if statusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code %d", statusCode) + } + + project := &Project{url: c.host, apiKey: c.apiKey} + err = json.Unmarshal(respBody, &project) + return project, err +} diff --git a/internal/stack/serverless/project.go b/internal/stack/serverless/project.go new file mode 100644 index 0000000000..9cfd68813f --- /dev/null +++ b/internal/stack/serverless/project.go @@ -0,0 +1,75 @@ +package serverless + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" +) + +// Project represents a serverless project +type Project struct { + url string + apiKey string + + Name string `json:"name"` + ID string `json:"id"` + Type string `json:"type"` + Region string `json:"region_id"` + + Credentials struct { + Username string `json:"username"` + Password string `json:"password"` + } `json:"credentials"` + + Endpoints struct { + Elasticsearch string `json:"elasticsearch"` + Kibana string `json:"kibana"` + Fleet string `json:"fleet,omitempty"` + APM string `json:"apm,omitempty"` + } `json:"endpoints"` +} + +// NewObservabilityProject creates a new observability type project +func NewObservabilityProject(ctx context.Context, url, name, apiKey, region string) (*Project, error) { + return newProject(ctx, url, name, apiKey, region, "observability") +} + +// newProject creates a new serverless project +// Note that the Project.Endpoints may not be populated and another call may be required. +func newProject(ctx context.Context, url, name, apiKey, region, projectType string) (*Project, error) { + ReqBody := struct { + Name string `json:"name"` + RegionID string `json:"region_id"` + }{ + Name: name, + RegionID: region, + } + p, err := json.Marshal(ReqBody) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, "POST", url+"/api/v1/serverless/projects/"+projectType, bytes.NewReader(p)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "ApiKey "+apiKey) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + p, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("unexpected status code %d, body: %s", resp.StatusCode, string(p)) + } + project := &Project{url: url, apiKey: apiKey} + + err = json.NewDecoder(resp.Body).Decode(project) + return project, err +} From 3f38c52274a3d6caf3f0a0f595028e78851f1150 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 27 Jul 2023 23:26:23 +0200 Subject: [PATCH 02/62] Ensure endpoints are healthy fleet server endpoint is built based on the elasticsearch endpoint fix creation --- internal/stack/serverless.go | 33 +++++- internal/stack/serverless/client.go | 63 +++++++++++- internal/stack/serverless/project.go | 148 +++++++++++++++++++++------ 3 files changed, 206 insertions(+), 38 deletions(-) diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 87f4d4db66..114e4aa287 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -1,8 +1,14 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package stack import ( + "context" "errors" "fmt" + "time" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" @@ -43,6 +49,12 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op return Config{}, fmt.Errorf("failed to create %s project %s in %s: %w", settings.Type, settings.Name, settings.Region, err) } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30) + defer cancel() + if err := sp.client.EnsureEndpoints(ctx, project); err != nil { + return Config{}, fmt.Errorf("failed to ensure endpoints have been provisioned properly: %w", err) + } + var config Config config.Provider = ProviderServerless config.Parameters = map[string]string{ @@ -53,6 +65,9 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op config.ElasticsearchHost = project.Endpoints.Elasticsearch config.KibanaHost = project.Endpoints.Kibana + config.ElasticsearchUsername = project.Credentials.Username + config.ElasticsearchPassword = project.Credentials.Password + printUserConfig(options.Printer, config) err = storeConfig(sp.profile, config) @@ -60,6 +75,12 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op return Config{}, fmt.Errorf("failed to store config: %w", err) } + logger.Debug("Waiting for creation plan to be completed") + err = project.EnsureHealthy(ctx) + if err != nil { + return Config{}, fmt.Errorf("not all services are healthy: %w", err) + } + return config, nil } @@ -150,6 +171,8 @@ func (sp *serverlessProvider) BootUp(options Options) error { // if err != nil { // return fmt.Errorf("failed to replace GeoIP databases: %w", err) // } + logger.Debugf("Project created: %s", project.Name) + printUserConfig(options.Printer, config) case nil: logger.Debugf("Project existed: %s", project.Name) printUserConfig(options.Printer, config) @@ -209,7 +232,7 @@ func (sp *serverlessProvider) TearDown(options Options) error { return fmt.Errorf("failed to find current project: %w", err) } - logger.Debugf("Deleting project %q", project.ID) + logger.Debugf("Deleting project %q (%s)", project.Name, project.ID) err = sp.deleteProject(project, options) if err != nil { @@ -222,10 +245,10 @@ func (sp *serverlessProvider) TearDown(options Options) error { // return fmt.Errorf("failed to delete GeoIP extension: %w", err) // } - err = storeConfig(sp.profile, Config{}) - if err != nil { - return fmt.Errorf("failed to store config: %w", err) - } + // err = storeConfig(sp.profile, Config{}) + // if err != nil { + // return fmt.Errorf("failed to store config: %w", err) + // } return nil } diff --git a/internal/stack/serverless/client.go b/internal/stack/serverless/client.go index 87fa903f85..912f3e275c 100644 --- a/internal/stack/serverless/client.go +++ b/internal/stack/serverless/client.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package serverless import ( @@ -10,6 +14,8 @@ import ( "net/http" "net/url" "os" + "strings" + "time" "github.com/elastic/elastic-package/internal/environment" "github.com/elastic/elastic-package/internal/logger" @@ -18,6 +24,9 @@ import ( type Client struct { host string apiKey string + + username string + password string } // ClientOption is functional option modifying Serverless API client. @@ -52,20 +61,34 @@ func NewClient(opts ...ClientOption) (*Client, error) { return c, nil } -// Address option sets the host to use to connect to Kibana. +// WithAddress option sets the host to use to connect to Kibana. func WithAddress(address string) ClientOption { return func(c *Client) { c.host = address } } -// Address option sets the host to use to connect to Kibana. +// WithApiKey option sets the host to use to connect to Kibana. func WithApiKey(apiKey string) ClientOption { return func(c *Client) { c.apiKey = apiKey } } +// WithUsername option sets the username. +func WithUsername(username string) ClientOption { + return func(c *Client) { + c.username = username + } +} + +// WithPassword option sets the password. +func WithPassword(password string) ClientOption { + return func(c *Client) { + c.password = password + } +} + func (c *Client) get(ctx context.Context, resourcePath string) (int, []byte, error) { return c.sendRequest(ctx, http.MethodGet, resourcePath, nil) } @@ -109,8 +132,13 @@ func (c *Client) newRequest(ctx context.Context, method, resourcePath string, re } req.Header.Add("content-type", "application/json") - req.Header.Add("Authorization", fmt.Sprintf("ApiKey %s", c.apiKey)) + if c.username != "" { + req.SetBasicAuth(c.username, c.password) + return req, nil + } + + req.Header.Add("Authorization", fmt.Sprintf("ApiKey %s", c.apiKey)) return req, nil } @@ -199,3 +227,32 @@ func (c *Client) GetProject(projectType, projectID string) (*Project, error) { err = json.Unmarshal(respBody, &project) return project, err } + +func (c *Client) EnsureEndpoints(ctx context.Context, project *Project) error { + timer := time.NewTimer(time.Millisecond) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } + + if project.Endpoints.Elasticsearch != "" { + if project.Endpoints.Fleet == "" { + logger.Debugf("Fleet Endpoint empty, setting it based on ES") + project.Endpoints.Fleet = strings.Replace(project.Endpoints.Elasticsearch, ".es.", ".fleet.", 1) + } + return nil + } + + newProject, err := c.GetProject(project.Type, project.ID) + if err != nil { + logger.Debugf("request error: %s", err.Error()) + timer.Reset(time.Second * 5) + continue + } + + project.Endpoints = newProject.Endpoints + timer.Reset(time.Second * 5) + } +} diff --git a/internal/stack/serverless/project.go b/internal/stack/serverless/project.go index 9cfd68813f..bac80bc8c7 100644 --- a/internal/stack/serverless/project.go +++ b/internal/stack/serverless/project.go @@ -1,12 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package serverless import ( - "bytes" "context" "encoding/json" "fmt" - "io" "net/http" + "time" + + "github.com/elastic/elastic-package/internal/logger" ) // Project represents a serverless project @@ -16,6 +21,7 @@ type Project struct { Name string `json:"name"` ID string `json:"id"` + Alias string `json:"alias"` Type string `json:"type"` Region string `json:"region_id"` @@ -32,44 +38,126 @@ type Project struct { } `json:"endpoints"` } -// NewObservabilityProject creates a new observability type project -func NewObservabilityProject(ctx context.Context, url, name, apiKey, region string) (*Project, error) { - return newProject(ctx, url, name, apiKey, region, "observability") +type serviceHealthy func(context.Context, *Project) error + +func (p *Project) EnsureHealthy(ctx context.Context) error { + if err := p.ensureServiceHealthy(ctx, getESHealthy); err != nil { + return fmt.Errorf("elasticsearch not healthy: %w", err) + } + if err := p.ensureServiceHealthy(ctx, getKibanaHealthy); err != nil { + return fmt.Errorf("kibana not healthy: %w", err) + } + if err := p.ensureServiceHealthy(ctx, getFleetHealthy); err != nil { + return fmt.Errorf("fleet not healthy: %w", err) + } + return nil } -// newProject creates a new serverless project -// Note that the Project.Endpoints may not be populated and another call may be required. -func newProject(ctx context.Context, url, name, apiKey, region, projectType string) (*Project, error) { - ReqBody := struct { - Name string `json:"name"` - RegionID string `json:"region_id"` - }{ - Name: name, - RegionID: region, - } - p, err := json.Marshal(ReqBody) +func (p *Project) ensureServiceHealthy(ctx context.Context, serviceFunc serviceHealthy) error { + timer := time.NewTimer(time.Millisecond) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } + + err := serviceFunc(ctx, p) + if err != nil { + logger.Debugf("service not ready: %s", err.Error()) + timer.Reset(time.Second * 5) + continue + } + + return nil + } + return nil +} + +func getESHealthy(ctx context.Context, project *Project) error { + client, err := NewClient( + WithAddress(project.Endpoints.Elasticsearch), + WithUsername(project.Credentials.Username), + WithPassword(project.Credentials.Password), + ) if err != nil { - return nil, err + return err } - req, err := http.NewRequestWithContext(ctx, "POST", url+"/api/v1/serverless/projects/"+projectType, bytes.NewReader(p)) + + statusCode, respBody, err := client.get(ctx, "/_cluster/health") if err != nil { - return nil, err + return fmt.Errorf("failed to query elasticsearch health: %w", err) + } + + if statusCode != http.StatusOK { + return fmt.Errorf("unexpected status code %d, body: %s", statusCode, string(respBody)) } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "ApiKey "+apiKey) - resp, err := http.DefaultClient.Do(req) + var health struct { + Status string `json:"status"` + } + if err := json.Unmarshal(respBody, &health); err != nil { + logger.Debugf("Unable to decode response: %v body: %s", err, string(respBody)) + return err + } + if health.Status == "green" { + return nil + } + return fmt.Errorf("elasticsearch unhealthy: %s", health.Status) +} + +func getKibanaHealthy(ctx context.Context, project *Project) error { + client, err := NewClient( + WithAddress(project.Endpoints.Kibana), + WithUsername(project.Credentials.Username), + WithPassword(project.Credentials.Password), + ) + if err != nil { + return err + } + + statusCode, respBody, err := client.get(ctx, "/api/status") + if err != nil { + return fmt.Errorf("failed to query kibana status: %w", err) + } + if statusCode != http.StatusOK { + return fmt.Errorf("unexpected status code %d, body: %s", statusCode, string(respBody)) + } + + var status struct { + Status struct { + Overall struct { + Level string `json:"level"` + } `json:"overall"` + } `json:"status"` + } + if err := json.Unmarshal(respBody, &status); err != nil { + logger.Debugf("Unable to decode response: %v body: %s", err, string(respBody)) + return err + } + if status.Status.Overall.Level == "available" { + return nil + } + return fmt.Errorf("kibana unhealthy: %s", status.Status.Overall.Level) +} + +func getFleetHealthy(ctx context.Context, project *Project) error { + client, err := NewClient( + WithAddress(project.Endpoints.Fleet), + WithUsername(project.Credentials.Username), + WithPassword(project.Credentials.Password), + ) if err != nil { - return nil, err + return err } - defer resp.Body.Close() - if resp.StatusCode != http.StatusCreated { - p, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("unexpected status code %d, body: %s", resp.StatusCode, string(p)) + statusCode, respBody, err := client.get(ctx, "/api/status") + if err != nil { + return fmt.Errorf("failed to query fleet status: %w", err) + } + if statusCode != http.StatusOK { + return fmt.Errorf("fleet unhealthy: status code %d, body: %s", statusCode, string(respBody)) } - project := &Project{url: url, apiKey: apiKey} - err = json.NewDecoder(resp.Body).Decode(project) - return project, err + return nil } From 4b3da639c1aec40ee6704687f6332f836c19a0c1 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 17 Aug 2023 13:23:55 +0200 Subject: [PATCH 03/62] Retrieve credentials --- internal/stack/serverless.go | 19 ++++++- internal/stack/serverless/client.go | 79 ++++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 114e4aa287..c21c964655 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -65,6 +65,24 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op config.ElasticsearchHost = project.Endpoints.Elasticsearch config.KibanaHost = project.Endpoints.Kibana + printUserConfig(options.Printer, config) + + err = storeConfig(sp.profile, config) + if err != nil { + return Config{}, fmt.Errorf("failed to store config: %w", err) + } + + logger.Debug("Waiting for creation plan to be completed") + err = sp.client.EnsureProjectInitialized(ctx, project) + if err != nil { + return Config{}, fmt.Errorf("project not initialized: %w", err) + } + + logger.Debugf("Getting credentials for project %s (%s)", project.Name, project.Type) + project, err = sp.client.ResetCredentials(ctx, project) + if err != nil { + return Config{}, fmt.Errorf("credentials not reset: %w", err) + } config.ElasticsearchUsername = project.Credentials.Username config.ElasticsearchPassword = project.Credentials.Password @@ -75,7 +93,6 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op return Config{}, fmt.Errorf("failed to store config: %w", err) } - logger.Debug("Waiting for creation plan to be completed") err = project.EnsureHealthy(ctx) if err != nil { return Config{}, fmt.Errorf("not all services are healthy: %w", err) diff --git a/internal/stack/serverless/client.go b/internal/stack/serverless/client.go index 912f3e275c..7806d88a45 100644 --- a/internal/stack/serverless/client.go +++ b/internal/stack/serverless/client.go @@ -192,6 +192,82 @@ func (c *Client) CreateProject(name, region, project string) (*Project, error) { return serverlessProject, err } +func (c *Client) EnsureProjectInitialized(ctx context.Context, project *Project) error { + timer := time.NewTimer(time.Millisecond) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } + + status, err := c.StatusProject(ctx, project) + if err != nil { + logger.Debugf("error querying for status: %s", err.Error()) + timer.Reset(time.Second * 5) + continue + } + + if status != "initialized" { + logger.Debugf("project not initialized, status: %s", status) + timer.Reset(time.Second * 5) + continue + } + + return nil + } + return nil +} + +func (c *Client) StatusProject(ctx context.Context, project *Project) (string, error) { + resourcePath := fmt.Sprintf("%s/api/v1/serverless/projects/%s/%s/status", c.host, project.Type, project.ID) + statusCode, respBody, err := c.get(ctx, resourcePath) + + if err != nil { + return "", fmt.Errorf("error getting status project: %w", err) + } + + if statusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code %d", statusCode) + } + + var status struct { + Phase string `json:"phase"` + } + + if err := json.Unmarshal(respBody, &status); err != nil { + return "", fmt.Errorf("unable to decode status: %w", err) + } + + return status.Phase, nil +} + +func (c *Client) ResetCredentials(ctx context.Context, project *Project) (*Project, error) { + resourcePath := fmt.Sprintf("%s/api/v1/serverless/projects/%s/%s/_reset-credentials", c.host, project.Type, project.ID) + statusCode, respBody, err := c.post(ctx, resourcePath, nil) + + if err != nil { + return nil, fmt.Errorf("error creating project: %w", err) + } + + if statusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code %d", statusCode) + } + + var credentials struct { + Username string `json:"username"` + Password string `json:"password"` + } + if err := json.Unmarshal(respBody, &credentials); err != nil { + return nil, fmt.Errorf("unable to decode credentials: %w", err) + } + + project.Credentials.Username = credentials.Username + project.Credentials.Password = credentials.Password + + return project, err +} + func (c *Client) DeleteProject(project *Project) error { ctx := context.Background() resourcePath := fmt.Sprintf("%s/api/v1/serverless/projects/%s/%s", c.host, project.Type, project.ID) @@ -201,7 +277,7 @@ func (c *Client) DeleteProject(project *Project) error { } if statusCode != http.StatusOK { - return fmt.Errorf("unexpected status code %ds", statusCode) + return fmt.Errorf("unexpected status code %d", statusCode) } return nil @@ -225,6 +301,7 @@ func (c *Client) GetProject(projectType, projectID string) (*Project, error) { project := &Project{url: c.host, apiKey: c.apiKey} err = json.Unmarshal(respBody, &project) + return project, err } From 21ce68785b7a7933d1e404df9ae9fc30c689c981 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 17 Aug 2023 15:48:30 +0200 Subject: [PATCH 04/62] Add status output --- internal/stack/serverless.go | 46 ++++++++++++++++++++++- internal/stack/serverless/client.go | 5 --- internal/stack/serverless/project.go | 56 ++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 6 deletions(-) diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index c21c964655..1b613daaf1 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -18,6 +18,7 @@ import ( const ( paramServerlessProjectID = "serverless_project_id" paramServerlessProjectType = "serverless_project_type" + paramServerlessFleetURL = "serverless_fleet_url" configRegion = "stack.serverless.region" configProjectType = "stack.serverless.type" @@ -86,6 +87,12 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op config.ElasticsearchUsername = project.Credentials.Username config.ElasticsearchPassword = project.Credentials.Password + config.Parameters[paramServerlessFleetURL], err = project.DefaultFleetServerURL(ctx) + if err != nil { + return Config{}, fmt.Errorf("failed to get fleet URL: %w", err) + } + project.Endpoints.Fleet = config.Parameters[paramServerlessFleetURL] + printUserConfig(options.Printer, config) err = storeConfig(sp.profile, config) @@ -124,6 +131,18 @@ func (sp *serverlessProvider) currentProject(config Config) (*serverless.Project return nil, fmt.Errorf("couldn't check project health: %w", err) } + project.Credentials.Username = config.ElasticsearchUsername + project.Credentials.Password = config.ElasticsearchPassword + + fleetURL, ok := config.Parameters[paramServerlessFleetURL] + if !ok { + fleetURL, err = project.DefaultFleetServerURL(context.Background()) + if err != nil { + return nil, fmt.Errorf("failed to get fleet URL: %w", err) + } + } + project.Endpoints.Fleet = fleetURL + return project, nil } @@ -280,5 +299,30 @@ func (sp *serverlessProvider) Dump(options DumpOptions) (string, error) { func (sp *serverlessProvider) Status(options Options) ([]ServiceStatus, error) { logger.Warn("Elastic Serverless provider is in technical preview") - return Status(options) + config, err := LoadConfig(sp.profile) + if err != nil { + return nil, fmt.Errorf("failed to load configuration: %w", err) + } + + project, err := sp.currentProject(config) + if err != nil { + return nil, err + } + + ctx := context.Background() + projectServiceStatus, err := project.Status(ctx) + if err != nil { + return nil, err + } + + var serviceStatus []ServiceStatus + for service, status := range projectServiceStatus { + serviceStatus = append(serviceStatus, ServiceStatus{ + Name: service, + Version: "serverless", + Status: status, + }) + } + + return serviceStatus, nil } diff --git a/internal/stack/serverless/client.go b/internal/stack/serverless/client.go index 7806d88a45..30fc71096a 100644 --- a/internal/stack/serverless/client.go +++ b/internal/stack/serverless/client.go @@ -14,7 +14,6 @@ import ( "net/http" "net/url" "os" - "strings" "time" "github.com/elastic/elastic-package/internal/environment" @@ -315,10 +314,6 @@ func (c *Client) EnsureEndpoints(ctx context.Context, project *Project) error { } if project.Endpoints.Elasticsearch != "" { - if project.Endpoints.Fleet == "" { - logger.Debugf("Fleet Endpoint empty, setting it based on ES") - project.Endpoints.Fleet = strings.Replace(project.Endpoints.Elasticsearch, ".es.", ".fleet.", 1) - } return nil } diff --git a/internal/stack/serverless/project.go b/internal/stack/serverless/project.go index bac80bc8c7..4566b1134c 100644 --- a/internal/stack/serverless/project.go +++ b/internal/stack/serverless/project.go @@ -7,6 +7,7 @@ package serverless import ( "context" "encoding/json" + "errors" "fmt" "net/http" "time" @@ -53,6 +54,23 @@ func (p *Project) EnsureHealthy(ctx context.Context) error { return nil } +func (p *Project) Status(ctx context.Context) (map[string]string, error) { + var status map[string]string + healthStatus := func(err error) string { + if err != nil { + return fmt.Sprintf("unhealthy: %s", err.Error()) + } + return "healthy" + } + + status = map[string]string{ + "elasticsearch": healthStatus(getESHealthy(ctx, p)), + "kibana": healthStatus(getKibanaHealthy(ctx, p)), + "fleet": healthStatus(getFleetHealthy(ctx, p)), + } + return status, nil +} + func (p *Project) ensureServiceHealthy(ctx context.Context, serviceFunc serviceHealthy) error { timer := time.NewTimer(time.Millisecond) for { @@ -74,6 +92,44 @@ func (p *Project) ensureServiceHealthy(ctx context.Context, serviceFunc serviceH return nil } +func (p *Project) DefaultFleetServerURL(ctx context.Context) (string, error) { + client, err := NewClient( + WithAddress(p.Endpoints.Kibana), + WithUsername(p.Credentials.Username), + WithPassword(p.Credentials.Password), + ) + if err != nil { + return "", err + } + statusCode, respBody, err := client.get(ctx, "/api/fleet/fleet_server_hosts") + if err != nil { + return "", fmt.Errorf("failed to query fleet server hosts: %w", err) + } + + if statusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code %d, body: %s", statusCode, string(respBody)) + } + + var hosts struct { + Items []struct { + IsDefault bool `json:"is_default"` + HostURLs []string `json:"host_urls"` + } `json:"items"` + } + err = json.Unmarshal(respBody, &hosts) + if err != nil { + return "", fmt.Errorf("failed to decode response: %w", err) + } + + for _, server := range hosts.Items { + if server.IsDefault && len(server.HostURLs) > 0 { + return server.HostURLs[0], nil + } + } + + return "", errors.New("could not find the fleet server URL for this project") +} + func getESHealthy(ctx context.Context, project *Project) error { client, err := NewClient( WithAddress(project.Endpoints.Elasticsearch), From ef5e47e4b67c2af4ecc91fe433e670aceb92305e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 17 Aug 2023 17:44:57 +0200 Subject: [PATCH 05/62] Request credentials when creating project --- internal/stack/serverless.go | 10 ++-------- internal/stack/serverless/client.go | 5 +++++ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 1b613daaf1..abb73f6b77 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -65,6 +65,8 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op config.ElasticsearchHost = project.Endpoints.Elasticsearch config.KibanaHost = project.Endpoints.Kibana + config.ElasticsearchUsername = project.Credentials.Username + config.ElasticsearchPassword = project.Credentials.Password printUserConfig(options.Printer, config) @@ -79,14 +81,6 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op return Config{}, fmt.Errorf("project not initialized: %w", err) } - logger.Debugf("Getting credentials for project %s (%s)", project.Name, project.Type) - project, err = sp.client.ResetCredentials(ctx, project) - if err != nil { - return Config{}, fmt.Errorf("credentials not reset: %w", err) - } - config.ElasticsearchUsername = project.Credentials.Username - config.ElasticsearchPassword = project.Credentials.Password - config.Parameters[paramServerlessFleetURL], err = project.DefaultFleetServerURL(ctx) if err != nil { return Config{}, fmt.Errorf("failed to get fleet URL: %w", err) diff --git a/internal/stack/serverless/client.go b/internal/stack/serverless/client.go index 30fc71096a..181843215a 100644 --- a/internal/stack/serverless/client.go +++ b/internal/stack/serverless/client.go @@ -185,6 +185,11 @@ func (c *Client) CreateProject(name, region, project string) (*Project, error) { serverlessProject := &Project{url: c.host, apiKey: c.apiKey} err = json.Unmarshal(respBody, &serverlessProject) + serverlessProject, err = c.ResetCredentials(ctx, serverlessProject) + if err != nil { + return nil, fmt.Errorf("failed to reset credentials: %w", err) + } + bytes, _ := json.MarshalIndent(&serverlessProject, "", " ") fmt.Printf("Project:\n%s", string(bytes)) From c04ee5de63c95d3aab50def82710b2ed0f2eb577 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 17 Aug 2023 17:46:24 +0200 Subject: [PATCH 06/62] Move package to internal --- internal/{stack => }/serverless/client.go | 0 internal/{stack => }/serverless/project.go | 0 internal/stack/serverless.go | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename internal/{stack => }/serverless/client.go (100%) rename internal/{stack => }/serverless/project.go (100%) diff --git a/internal/stack/serverless/client.go b/internal/serverless/client.go similarity index 100% rename from internal/stack/serverless/client.go rename to internal/serverless/client.go diff --git a/internal/stack/serverless/project.go b/internal/serverless/project.go similarity index 100% rename from internal/stack/serverless/project.go rename to internal/serverless/project.go diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index abb73f6b77..2d97dd2532 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -12,7 +12,7 @@ import ( "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" - "github.com/elastic/elastic-package/internal/stack/serverless" + "github.com/elastic/elastic-package/internal/serverless" ) const ( From f4632152ecc58a68b09d16b7df20b57273e8ab4b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 17 Aug 2023 19:08:59 +0200 Subject: [PATCH 07/62] Add elastic agent to serverless project --- internal/serverless/project.go | 2 +- .../_static/serverless-elastic-agent.env.tmpl | 10 + .../_static/serverless-elastic-agent.yml.tmpl | 26 ++ internal/stack/serverless.go | 259 +++++++++++++++--- internal/stack/serverlessresources.go | 75 +++++ 5 files changed, 334 insertions(+), 38 deletions(-) create mode 100644 internal/stack/_static/serverless-elastic-agent.env.tmpl create mode 100644 internal/stack/_static/serverless-elastic-agent.yml.tmpl create mode 100644 internal/stack/serverlessresources.go diff --git a/internal/serverless/project.go b/internal/serverless/project.go index 4566b1134c..9a9166d185 100644 --- a/internal/serverless/project.go +++ b/internal/serverless/project.go @@ -156,7 +156,7 @@ func getESHealthy(ctx context.Context, project *Project) error { logger.Debugf("Unable to decode response: %v body: %s", err, string(respBody)) return err } - if health.Status == "green" { + if health.Status == "green" || health.Status == "yellow" { return nil } return fmt.Errorf("elasticsearch unhealthy: %s", health.Status) diff --git a/internal/stack/_static/serverless-elastic-agent.env.tmpl b/internal/stack/_static/serverless-elastic-agent.env.tmpl new file mode 100644 index 0000000000..d8888122e1 --- /dev/null +++ b/internal/stack/_static/serverless-elastic-agent.env.tmpl @@ -0,0 +1,10 @@ +{{ $version := fact "agent_version" }} +FLEET_ENROLL=1 +FLEET_URL={{ fact "fleet_url" }} +KIBANA_FLEET_HOST={{ fact "kibana_host" }} +KIBANA_HOST={{ fact "kibana_host" }} +ELASTICSEARCH_USERNAME={{ fact "username" }} +ELASTICSEARCH_PASSWORD={{ fact "password" }} +{{ if not (semverLessThan $version "8.0.0") }} +FLEET_TOKEN_POLICY_NAME=Elastic-Agent (elastic-package) +{{ end }} diff --git a/internal/stack/_static/serverless-elastic-agent.yml.tmpl b/internal/stack/_static/serverless-elastic-agent.yml.tmpl new file mode 100644 index 0000000000..150cff7ccb --- /dev/null +++ b/internal/stack/_static/serverless-elastic-agent.yml.tmpl @@ -0,0 +1,26 @@ +version: '2.3' +services: + elastic-agent: + image: "{{ fact "agent_image" }}" + healthcheck: + test: "elastic-agent status" + timeout: 2s + start_period: 360s + retries: 180 + interval: 5s + hostname: docker-fleet-agent + env_file: "./serverless-elastic-agent.env" + volumes: + - type: bind + source: ../../../tmp/service_logs/ + target: /tmp/service_logs/ + # Mount service_logs under /run too as a testing workaround for the journald input (see elastic-package#1235). + - type: bind + source: ../../../tmp/service_logs/ + target: /run/service_logs/ + + elastic-agent_is_ready: + image: tianon/true + depends_on: + elastic-agent: + condition: service_healthy diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 2d97dd2532..08add523d2 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -6,10 +6,17 @@ package stack import ( "context" + "encoding/json" "errors" "fmt" + "io" + "net/http" + "net/url" + "strings" "time" + "github.com/elastic/elastic-package/internal/compose" + "github.com/elastic/elastic-package/internal/docker" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" "github.com/elastic/elastic-package/internal/serverless" @@ -190,11 +197,11 @@ func (sp *serverlessProvider) BootUp(options Options) error { return fmt.Errorf("failed to create deployment: %w", err) } - // logger.Infof("Creating agent policy") - // err = sp.createAgentPolicy(config, options.StackVersion) - // if err != nil { - // return fmt.Errorf("failed to create agent policy: %w", err) - // } + logger.Infof("Creating agent policy") + err = sp.createAgentPolicy(config, options.StackVersion) + if err != nil { + return fmt.Errorf("failed to create agent policy: %w", err) + } // logger.Infof("Replacing GeoIP databases") // err = cp.replaceGeoIPDatabases(config, options, settings.TemplateID, settings.Region, payload.Resources.Elasticsearch[0].Plan.ClusterTopology) @@ -213,38 +220,163 @@ func (sp *serverlessProvider) BootUp(options Options) error { // } } - // logger.Infof("Starting local agent") - // err = sp.startLocalAgent(options, config) - // if err != nil { - // return fmt.Errorf("failed to start local agent: %w", err) - // } + logger.Infof("Starting local agent") + err = sp.startLocalAgent(options, config) + if err != nil { + return fmt.Errorf("failed to start local agent: %w", err) + } + + return nil +} + +func (sp *serverlessProvider) composeProjectName() string { + return DockerComposeProjectName(sp.profile) +} + +func (sp *serverlessProvider) localAgentComposeProject() (*compose.Project, error) { + composeFile := sp.profile.Path(profileStackPath, ServerlessComposeFile) + return compose.NewProject(sp.composeProjectName(), composeFile) +} + +func (sp *serverlessProvider) startLocalAgent(options Options, config Config) error { + err := applyServerlessResources(sp.profile, options.StackVersion, config) + if err != nil { + return fmt.Errorf("could not initialize compose files for local agent: %w", err) + } + + project, err := sp.localAgentComposeProject() + if err != nil { + return fmt.Errorf("could not initialize local agent compose project") + } + + err = project.Build(compose.CommandOptions{}) + if err != nil { + return fmt.Errorf("failed to build images for local agent: %w", err) + } + + err = project.Up(compose.CommandOptions{ExtraArgs: []string{"-d"}}) + if err != nil { + return fmt.Errorf("failed to start local agent: %w", err) + } + + return nil +} + +const serverlessKibanaAgentPolicy = `{ + "name": "Elastic-Agent (elastic-package)", + "id": "elastic-agent-managed-ep", + "description": "Policy created by elastic-package", + "namespace": "default", + "monitoring_enabled": [ + "logs", + "metrics" + ] +}` + +const serverlessKibanaPackagePolicy = `{ + "name": "system-1", + "policy_id": "elastic-agent-managed-ep", + "package": { + "name": "system", + "version": "%s" + } +}` + +func doKibanaRequest(config Config, req *http.Request) error { + req.SetBasicAuth(config.ElasticsearchUsername, config.ElasticsearchPassword) + req.Header.Add("content-type", "application/json") + req.Header.Add("kbn-xsrf", "elastic-package") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("performing request failed: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusConflict { + // Already created, go on. + // TODO: We could try to update the policy. + return nil + } + if resp.StatusCode >= 300 { + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("request failed with status %v and could not read body: %w", resp.StatusCode, err) + } + return fmt.Errorf("request failed with status %v and response %v", resp.StatusCode, string(body)) + } + return nil +} + +func (sp *serverlessProvider) createAgentPolicy(config Config, stackVersion string) error { + agentPoliciesURL, err := url.JoinPath(config.KibanaHost, "/api/fleet/agent_policies") + if err != nil { + return fmt.Errorf("failed to build url for agent policies: %w", err) + } + req, err := http.NewRequest(http.MethodPost, agentPoliciesURL, strings.NewReader(serverlessKibanaAgentPolicy)) + if err != nil { + return fmt.Errorf("failed to initialize request to create agent policy: %w", err) + } + err = doKibanaRequest(config, req) + if err != nil { + return fmt.Errorf("error while creating agent policy: %w", err) + } + + systemVersion, err := getPackageVersion("https://epr.elastic.co", "system", stackVersion) + if err != nil { + return fmt.Errorf("could not get the system package version for kibana %v: %w", stackVersion, err) + } + + packagePoliciesURL, err := url.JoinPath(config.KibanaHost, "/api/fleet/package_policies") + if err != nil { + return fmt.Errorf("failed to build url for package policies: %w", err) + } + packagePolicy := fmt.Sprintf(serverlessKibanaPackagePolicy, systemVersion) + req, err = http.NewRequest(http.MethodPost, packagePoliciesURL, strings.NewReader(packagePolicy)) + if err != nil { + return fmt.Errorf("failed to initialize request to create package policy: %w", err) + } + err = doKibanaRequest(config, req) + if err != nil { + return fmt.Errorf("error while creating package policy: %w", err) + } return nil } -// func (sp *serverlessProvider) startLocalAgent(options Options, config Config) error { -// err := applyCloudResources(sp.profile, options.StackVersion, config) -// if err != nil { -// return fmt.Errorf("could not initialize compose files for local agent: %w", err) -// } -// -// project, err := sp.localAgentComposeProject() -// if err != nil { -// return fmt.Errorf("could not initialize local agent compose project") -// } -// -// err = project.Build(compose.CommandOptions{}) -// if err != nil { -// return fmt.Errorf("failed to build images for local agent: %w", err) -// } -// -// err = project.Up(compose.CommandOptions{ExtraArgs: []string{"-d"}}) -// if err != nil { -// return fmt.Errorf("failed to start local agent: %w", err) -// } -// -// return nil -// } +func getPackageVersion(registryURL, packageName, stackVersion string) (string, error) { + searchURL, err := url.JoinPath(registryURL, "search") + if err != nil { + return "", fmt.Errorf("could not build URL: %w", err) + } + searchURL = fmt.Sprintf("%s?package=%s&kibana.version=%s", searchURL, packageName, stackVersion) + resp, err := http.Get(searchURL) + if err != nil { + return "", fmt.Errorf("request failed (url: %s): %w", searchURL, err) + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + return "", fmt.Errorf("unexpected status code %v", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read response body: %w", err) + } + var packages []struct { + Name string `json:"name"` + Version string `json:"version"` + } + err = json.Unmarshal(body, &packages) + if err != nil { + return "", fmt.Errorf("failed to parse response body: %w", err) + } + if len(packages) != 1 { + return "", fmt.Errorf("expected 1 package, obtained %v", len(packages)) + } + if found := packages[0].Name; found != packageName { + return "", fmt.Errorf("expected package %s, found %s", packageName, found) + } + + return packages[0].Version, nil +} func (sp *serverlessProvider) TearDown(options Options) error { config, err := LoadConfig(sp.profile) @@ -252,10 +384,10 @@ func (sp *serverlessProvider) TearDown(options Options) error { return fmt.Errorf("failed to load configuration: %w", err) } - // err = cp.destroyLocalAgent() - // if err != nil { - // return fmt.Errorf("failed to destroy local agent: %w", err) - // } + err = sp.destroyLocalAgent() + if err != nil { + return fmt.Errorf("failed to destroy local agent: %w", err) + } project, err := sp.currentProject(config) if err != nil { @@ -283,6 +415,20 @@ func (sp *serverlessProvider) TearDown(options Options) error { return nil } +func (sp *serverlessProvider) destroyLocalAgent() error { + project, err := sp.localAgentComposeProject() + if err != nil { + return fmt.Errorf("could not initialize local agent compose project") + } + + err = project.Down(compose.CommandOptions{}) + if err != nil { + return fmt.Errorf("failed to destroy local agent: %w", err) + } + + return nil +} + func (sp *serverlessProvider) Update(options Options) error { return fmt.Errorf("not implemented") } @@ -318,5 +464,44 @@ func (sp *serverlessProvider) Status(options Options) ([]ServiceStatus, error) { }) } + agentStatus, err := sp.localAgentStatus() + if err != nil { + return nil, fmt.Errorf("failed to get local agent status: %w", err) + } + + serviceStatus = append(serviceStatus, agentStatus...) + return serviceStatus, nil } + +func (sp *serverlessProvider) localAgentStatus() ([]ServiceStatus, error) { + var services []ServiceStatus + // query directly to docker to avoid load environment variables (e.g. STACK_VERSION_VARIANT) and profiles + containerIDs, err := docker.ContainerIDsWithLabel(projectLabelDockerCompose, sp.composeProjectName()) + if err != nil { + return nil, err + } + + if len(containerIDs) == 0 { + return services, nil + } + + containerDescriptions, err := docker.InspectContainers(containerIDs...) + if err != nil { + return nil, err + } + + for _, containerDescription := range containerDescriptions { + service, err := newServiceStatus(&containerDescription) + if err != nil { + return nil, err + } + if strings.HasSuffix(service.Name, readyServicesSuffix) { + continue + } + logger.Debugf("Adding Service: \"%v\"", service.Name) + services = append(services, *service) + } + + return services, nil +} diff --git a/internal/stack/serverlessresources.go b/internal/stack/serverlessresources.go new file mode 100644 index 0000000000..8ce61be923 --- /dev/null +++ b/internal/stack/serverlessresources.go @@ -0,0 +1,75 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stack + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/elastic/elastic-package/internal/install" + "github.com/elastic/elastic-package/internal/profile" + "github.com/elastic/go-resource" +) + +const ( + // ServerlessElasticAgentEnvFile is the elastic agent environment variables file for the + // serverless provider. + ServerlessElasticAgentEnvFile = "serverless-elastic-agent.env" + + // ServerlessComposeFile is the docker-compose snapshot.yml file name. + ServerlessComposeFile = "serverless-elastic-agent.yml" +) + +var ( + serverlessStackResources = []resource.Resource{ + &resource.File{ + Path: ServerlessComposeFile, + Content: staticSource.Template("_static/serverless-elastic-agent.yml.tmpl"), + }, + &resource.File{ + Path: ServerlessElasticAgentEnvFile, + Content: staticSource.Template("_static/serverless-elastic-agent.env.tmpl"), + }, + } +) + +func applyServerlessResources(profile *profile.Profile, stackVersion string, config Config) error { + appConfig, err := install.Configuration() + if err != nil { + return fmt.Errorf("can't read application configuration: %w", err) + } + + stackDir := filepath.Join(profile.ProfilePath, profileStackPath) + + resourceManager := resource.NewManager() + resourceManager.AddFacter(resource.StaticFacter{ + "agent_version": stackVersion, + "agent_image": appConfig.StackImageRefs(stackVersion).ElasticAgent, + "username": config.ElasticsearchUsername, + "password": config.ElasticsearchPassword, + "kibana_host": config.KibanaHost, + "fleet_url": config.Parameters[paramServerlessFleetURL], + }) + + os.MkdirAll(stackDir, 0755) + resourceManager.RegisterProvider("file", &resource.FileProvider{ + Prefix: stackDir, + }) + + results, err := resourceManager.Apply(serverlessStackResources) + if err != nil { + var errors []string + for _, result := range results { + if err := result.Err(); err != nil { + errors = append(errors, err.Error()) + } + } + return fmt.Errorf("%w: %s", err, strings.Join(errors, ", ")) + } + + return nil +} From a54a1aae6a4add30a04b3d6a6df2fb0327c62f6e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 17 Aug 2023 19:40:32 +0200 Subject: [PATCH 08/62] Remove debug outputs --- internal/stack/serverless.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 08add523d2..2a87a64380 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -75,8 +75,8 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op config.ElasticsearchUsername = project.Credentials.Username config.ElasticsearchPassword = project.Credentials.Password - printUserConfig(options.Printer, config) - + // Store config now in case fails initialization or other requests, + // so it can be destroyed later err = storeConfig(sp.profile, config) if err != nil { return Config{}, fmt.Errorf("failed to store config: %w", err) @@ -96,6 +96,7 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op printUserConfig(options.Printer, config) + // update config with latest updates (e.g. fleet server url) err = storeConfig(sp.profile, config) if err != nil { return Config{}, fmt.Errorf("failed to store config: %w", err) @@ -208,8 +209,6 @@ func (sp *serverlessProvider) BootUp(options Options) error { // if err != nil { // return fmt.Errorf("failed to replace GeoIP databases: %w", err) // } - logger.Debugf("Project created: %s", project.Name) - printUserConfig(options.Printer, config) case nil: logger.Debugf("Project existed: %s", project.Name) printUserConfig(options.Printer, config) From 7f091eb6cd8412845f09f916a54cba6664a04f90 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 21 Aug 2023 16:02:04 +0200 Subject: [PATCH 09/62] Create elasticsearch client in stack --- cmd/benchmark.go | 6 +-- cmd/dump.go | 3 +- cmd/testrunner.go | 4 +- internal/dump/installedobjects_test.go | 4 +- internal/elasticsearch/client.go | 30 +++++++------ internal/elasticsearch/error.go | 3 ++ internal/elasticsearch/test/httptest.go | 5 ++- internal/serverless/project.go | 60 ++++++++++++------------- internal/stack/clients.go | 26 +++++++++++ internal/stack/serverless.go | 25 +++++++++++ 10 files changed, 110 insertions(+), 56 deletions(-) create mode 100644 internal/stack/clients.go diff --git a/cmd/benchmark.go b/cmd/benchmark.go index 2e14384fdf..1f80f7436e 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -15,6 +15,7 @@ import ( "github.com/elastic/elastic-package/internal/corpusgenerator" "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/kibana" + "github.com/elastic/elastic-package/internal/stack" "github.com/spf13/cobra" @@ -25,7 +26,6 @@ import ( "github.com/elastic/elastic-package/internal/benchrunner/runners/system" "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/common" - "github.com/elastic/elastic-package/internal/elasticsearch" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/signal" "github.com/elastic/elastic-package/internal/testrunner" @@ -161,7 +161,7 @@ func pipelineCommandAction(cmd *cobra.Command, args []string) error { return errors.New("no pipeline benchmarks found") } - esClient, err := elasticsearch.NewClient() + esClient, err := stack.NewElasticsearchClient() if err != nil { return fmt.Errorf("can't create Elasticsearch client: %w", err) } @@ -258,7 +258,7 @@ func systemCommandAction(cmd *cobra.Command, args []string) error { signal.Enable() - esClient, err := elasticsearch.NewClient() + esClient, err := stack.NewElasticsearchClient() if err != nil { return fmt.Errorf("can't create Elasticsearch client: %w", err) } diff --git a/cmd/dump.go b/cmd/dump.go index 939b52170b..6768f0f750 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -13,6 +13,7 @@ import ( "github.com/elastic/elastic-package/internal/dump" "github.com/elastic/elastic-package/internal/elasticsearch" "github.com/elastic/elastic-package/internal/kibana" + "github.com/elastic/elastic-package/internal/stack" ) const dumpLongDescription = `Use this command as an exploratory tool to dump resources from Elastic Stack (objects installed as part of package and agent policies).` @@ -79,7 +80,7 @@ func dumpInstalledObjectsCmdAction(cmd *cobra.Command, args []string) error { if tlsSkipVerify { clientOptions = append(clientOptions, elasticsearch.OptionWithSkipTLSVerify()) } - client, err := elasticsearch.NewClient(clientOptions...) + client, err := stack.NewElasticsearchClient(clientOptions...) if err != nil { return fmt.Errorf("failed to initialize Elasticsearch client: %w", err) } diff --git a/cmd/testrunner.go b/cmd/testrunner.go index 3880fe4fd1..73c33c7cb5 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -15,10 +15,10 @@ import ( "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/common" - "github.com/elastic/elastic-package/internal/elasticsearch" "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/signal" + "github.com/elastic/elastic-package/internal/stack" "github.com/elastic/elastic-package/internal/testrunner" "github.com/elastic/elastic-package/internal/testrunner/reporters/formats" "github.com/elastic/elastic-package/internal/testrunner/reporters/outputs" @@ -214,7 +214,7 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command return err } - esClient, err := elasticsearch.NewClient() + esClient, err := stack.NewElasticsearchClient() if err != nil { return fmt.Errorf("can't create Elasticsearch client: %w", err) } diff --git a/internal/dump/installedobjects_test.go b/internal/dump/installedobjects_test.go index d83d5a0df4..fd6e62b4ef 100644 --- a/internal/dump/installedobjects_test.go +++ b/internal/dump/installedobjects_test.go @@ -17,9 +17,9 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/elastic/elastic-package/internal/elasticsearch" estest "github.com/elastic/elastic-package/internal/elasticsearch/test" "github.com/elastic/elastic-package/internal/files" + "github.com/elastic/elastic-package/internal/stack" ) func TestDumpInstalledObjects(t *testing.T) { @@ -64,7 +64,7 @@ type installedObjectsDumpSuite struct { func (s *installedObjectsDumpSuite) SetupTest() { _, err := os.Stat(s.DumpDir) if errors.Is(err, os.ErrNotExist) { - client, err := elasticsearch.NewClient() + client, err := stack.NewElasticsearchClient() s.Require().NoError(err) dumper := NewInstalledObjectsDumper(client.API, s.PackageName) diff --git a/internal/elasticsearch/client.go b/internal/elasticsearch/client.go index 4b6a8ae070..ba43aa33e4 100644 --- a/internal/elasticsearch/client.go +++ b/internal/elasticsearch/client.go @@ -12,14 +12,12 @@ import ( "fmt" "io" "net/http" - "os" "strings" "github.com/elastic/go-elasticsearch/v7" "github.com/elastic/go-elasticsearch/v7/esapi" "github.com/elastic/elastic-package/internal/certs" - "github.com/elastic/elastic-package/internal/stack" ) // API contains the elasticsearch APIs @@ -47,16 +45,6 @@ type clientOptions struct { skipTLSVerify bool } -// defaultOptionsFromEnv returns clientOptions initialized with values from environmet variables. -func defaultOptionsFromEnv() clientOptions { - return clientOptions{ - address: os.Getenv(stack.ElasticsearchHostEnv), - username: os.Getenv(stack.ElasticsearchUsernameEnv), - password: os.Getenv(stack.ElasticsearchPasswordEnv), - certificateAuthority: os.Getenv(stack.CACertificateEnv), - } -} - type ClientOption func(*clientOptions) // OptionWithAddress sets the address to be used by the client. @@ -66,6 +54,20 @@ func OptionWithAddress(address string) ClientOption { } } +// OptionWithUsername sets the username to be used by the client. +func OptionWithUsername(username string) ClientOption { + return func(opts *clientOptions) { + opts.username = username + } +} + +// OptionWithPassword sets the password to be used by the client. +func OptionWithPassword(password string) ClientOption { + return func(opts *clientOptions) { + opts.password = password + } +} + // OptionWithCertificateAuthority sets the certificate authority to be used by the client. func OptionWithCertificateAuthority(certificateAuthority string) ClientOption { return func(opts *clientOptions) { @@ -87,13 +89,13 @@ type Client struct { // NewClient method creates new instance of the Elasticsearch client. func NewClient(customOptions ...ClientOption) (*Client, error) { - options := defaultOptionsFromEnv() + options := clientOptions{} for _, option := range customOptions { option(&options) } if options.address == "" { - return nil, stack.UndefinedEnvError(stack.ElasticsearchHostEnv) + return nil, ErrUndefinedAddress } config := elasticsearch.Config{ diff --git a/internal/elasticsearch/error.go b/internal/elasticsearch/error.go index fb717f300b..0385fdfc6b 100644 --- a/internal/elasticsearch/error.go +++ b/internal/elasticsearch/error.go @@ -7,6 +7,7 @@ package elasticsearch import ( "bytes" "encoding/json" + "errors" "fmt" ) @@ -74,3 +75,5 @@ func NewError(body []byte) error { // Fall back to including to raw body if it cannot be parsed. return fmt.Errorf("elasticsearch error: %v", string(body)) } + +var ErrUndefinedAddress = errors.New("missing elasticsearch address") diff --git a/internal/elasticsearch/test/httptest.go b/internal/elasticsearch/test/httptest.go index 486e683276..fffa49de05 100644 --- a/internal/elasticsearch/test/httptest.go +++ b/internal/elasticsearch/test/httptest.go @@ -16,6 +16,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/stack" ) // NewClient returns a client for a testing http server that uses prerecorded @@ -26,7 +27,7 @@ func NewClient(t *testing.T, serverDataDir string) *elasticsearch.Client { server := testElasticsearchServer(t, serverDataDir) t.Cleanup(func() { server.Close() }) - client, err := elasticsearch.NewClient( + client, err := stack.NewElasticsearchClient( elasticsearch.OptionWithAddress(server.URL), ) require.NoError(t, err) @@ -56,7 +57,7 @@ func pathForURL(url string) string { } func recordRequest(t *testing.T, r *http.Request, path string) { - client, err := elasticsearch.NewClient() + client, err := stack.NewElasticsearchClient() require.NoError(t, err) t.Logf("Recording %s in %s", r.URL.Path, path) diff --git a/internal/serverless/project.go b/internal/serverless/project.go index 9a9166d185..0111e322d6 100644 --- a/internal/serverless/project.go +++ b/internal/serverless/project.go @@ -12,6 +12,7 @@ import ( "net/http" "time" + "github.com/elastic/elastic-package/internal/elasticsearch" "github.com/elastic/elastic-package/internal/logger" ) @@ -37,12 +38,14 @@ type Project struct { Fleet string `json:"fleet,omitempty"` APM string `json:"apm,omitempty"` } `json:"endpoints"` + + ElasticsearchClient *elasticsearch.Client } type serviceHealthy func(context.Context, *Project) error func (p *Project) EnsureHealthy(ctx context.Context) error { - if err := p.ensureServiceHealthy(ctx, getESHealthy); err != nil { + if err := p.ensureElasticserchHealthy(ctx); err != nil { return fmt.Errorf("elasticsearch not healthy: %w", err) } if err := p.ensureServiceHealthy(ctx, getKibanaHealthy); err != nil { @@ -64,13 +67,34 @@ func (p *Project) Status(ctx context.Context) (map[string]string, error) { } status = map[string]string{ - "elasticsearch": healthStatus(getESHealthy(ctx, p)), + "elasticsearch": healthStatus(p.getESHealth(ctx)), "kibana": healthStatus(getKibanaHealthy(ctx, p)), "fleet": healthStatus(getFleetHealthy(ctx, p)), } return status, nil } +func (p *Project) ensureElasticserchHealthy(ctx context.Context) error { + timer := time.NewTimer(time.Millisecond) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } + + err := p.ElasticsearchClient.CheckHealth(ctx) + if err != nil { + logger.Debugf("service not ready: %s", err.Error()) + timer.Reset(time.Second * 5) + continue + } + + return nil + } + return nil +} + func (p *Project) ensureServiceHealthy(ctx context.Context, serviceFunc serviceHealthy) error { timer := time.NewTimer(time.Millisecond) for { @@ -130,36 +154,8 @@ func (p *Project) DefaultFleetServerURL(ctx context.Context) (string, error) { return "", errors.New("could not find the fleet server URL for this project") } -func getESHealthy(ctx context.Context, project *Project) error { - client, err := NewClient( - WithAddress(project.Endpoints.Elasticsearch), - WithUsername(project.Credentials.Username), - WithPassword(project.Credentials.Password), - ) - if err != nil { - return err - } - - statusCode, respBody, err := client.get(ctx, "/_cluster/health") - if err != nil { - return fmt.Errorf("failed to query elasticsearch health: %w", err) - } - - if statusCode != http.StatusOK { - return fmt.Errorf("unexpected status code %d, body: %s", statusCode, string(respBody)) - } - - var health struct { - Status string `json:"status"` - } - if err := json.Unmarshal(respBody, &health); err != nil { - logger.Debugf("Unable to decode response: %v body: %s", err, string(respBody)) - return err - } - if health.Status == "green" || health.Status == "yellow" { - return nil - } - return fmt.Errorf("elasticsearch unhealthy: %s", health.Status) +func (p *Project) getESHealth(ctx context.Context) error { + return p.ElasticsearchClient.CheckHealth(ctx) } func getKibanaHealthy(ctx context.Context, project *Project) error { diff --git a/internal/stack/clients.go b/internal/stack/clients.go new file mode 100644 index 0000000000..0cadcb0b6a --- /dev/null +++ b/internal/stack/clients.go @@ -0,0 +1,26 @@ +package stack + +import ( + "errors" + "os" + + "github.com/elastic/elastic-package/internal/elasticsearch" +) + +func NewElasticsearchClient(customOptions ...elasticsearch.ClientOption) (*elasticsearch.Client, error) { + + options := []elasticsearch.ClientOption{ + elasticsearch.OptionWithAddress(os.Getenv(ElasticsearchHostEnv)), + elasticsearch.OptionWithPassword(os.Getenv(ElasticsearchPasswordEnv)), + elasticsearch.OptionWithUsername(os.Getenv(ElasticsearchUsernameEnv)), + elasticsearch.OptionWithCertificateAuthority(os.Getenv(CACertificateEnv)), + } + options = append(options, customOptions...) + client, err := elasticsearch.NewClient(options...) + + if errors.Is(err, elasticsearch.ErrUndefinedAddress) { + return nil, UndefinedEnvError(ElasticsearchHostEnv) + } + + return client, err +} diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 2a87a64380..eab50c4753 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -17,6 +17,7 @@ import ( "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/docker" + "github.com/elastic/elastic-package/internal/elasticsearch" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" "github.com/elastic/elastic-package/internal/serverless" @@ -88,6 +89,11 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op return Config{}, fmt.Errorf("project not initialized: %w", err) } + project, err = sp.createClients(project) + if err != nil { + return Config{}, fmt.Errorf("failed to create project client") + } + config.Parameters[paramServerlessFleetURL], err = project.DefaultFleetServerURL(ctx) if err != nil { return Config{}, fmt.Errorf("failed to get fleet URL: %w", err) @@ -110,6 +116,20 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op return config, nil } +func (sp *serverlessProvider) createClients(project *serverless.Project) (*serverless.Project, error) { + var err error + project.ElasticsearchClient, err = NewElasticsearchClient( + elasticsearch.OptionWithAddress(project.Endpoints.Elasticsearch), + elasticsearch.OptionWithUsername(project.Credentials.Username), + elasticsearch.OptionWithPassword(project.Credentials.Password), + ) + if err != nil { + return project, fmt.Errorf("failed to create elasticsearch client") + } + + return project, nil +} + func (sp *serverlessProvider) deleteProject(project *serverless.Project, options Options) error { return sp.client.DeleteProject(project) } @@ -145,6 +165,11 @@ func (sp *serverlessProvider) currentProject(config Config) (*serverless.Project } project.Endpoints.Fleet = fleetURL + project, err = sp.createClients(project) + if err != nil { + return nil, fmt.Errorf("failed to create project client") + } + return project, nil } From 98acfbd915a59f6d3485481f29b5202a10253fbe Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 21 Aug 2023 16:59:35 +0200 Subject: [PATCH 10/62] Create kibana client in stack --- cmd/benchmark.go | 3 +- cmd/dump.go | 2 +- cmd/export.go | 3 +- cmd/install.go | 4 +-- cmd/uninstall.go | 4 +-- internal/dump/agentpolicies_test.go | 7 ++-- internal/kibana/client.go | 34 +++++++++++-------- internal/kibana/status.go | 28 +++++++++++++++ internal/stack/clients.go | 18 ++++++++++ internal/testrunner/runners/asset/runner.go | 4 +-- .../system/servicedeployer/custom_agent.go | 3 +- .../system/servicedeployer/kubernetes.go | 3 +- 12 files changed, 81 insertions(+), 32 deletions(-) diff --git a/cmd/benchmark.go b/cmd/benchmark.go index 1f80f7436e..065295e8ff 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -14,7 +14,6 @@ import ( "github.com/elastic/elastic-package/internal/corpusgenerator" "github.com/elastic/elastic-package/internal/install" - "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/stack" "github.com/spf13/cobra" @@ -267,7 +266,7 @@ func systemCommandAction(cmd *cobra.Command, args []string) error { return err } - kc, err := kibana.NewClient() + kc, err := stack.NewKibanaClient() if err != nil { return fmt.Errorf("can't create Kibana client: %w", err) } diff --git a/cmd/dump.go b/cmd/dump.go index 6768f0f750..40ef778767 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -120,7 +120,7 @@ func dumpAgentPoliciesCmdAction(cmd *cobra.Command, args []string) error { if tlsSkipVerify { clientOptions = append(clientOptions, kibana.TLSSkipVerify()) } - kibanaClient, err := kibana.NewClient(clientOptions...) + kibanaClient, err := stack.NewKibanaClient(clientOptions...) if err != nil { return fmt.Errorf("failed to initialize Kibana client: %w", err) } diff --git a/cmd/export.go b/cmd/export.go index b72bb8a1c1..19ee8d009d 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -15,6 +15,7 @@ import ( "github.com/elastic/elastic-package/internal/common" "github.com/elastic/elastic-package/internal/export" "github.com/elastic/elastic-package/internal/kibana" + "github.com/elastic/elastic-package/internal/stack" ) const exportLongDescription = `Use this command to export assets relevant for the package, e.g. Kibana dashboards.` @@ -65,7 +66,7 @@ func exportDashboardsCmd(cmd *cobra.Command, args []string) error { return cobraext.FlagParsingError(err, cobraext.AllowSnapshotFlagName) } - kibanaClient, err := kibana.NewClient(opts...) + kibanaClient, err := stack.NewKibanaClient(opts...) if err != nil { return fmt.Errorf("can't create Kibana client: %w", err) } diff --git a/cmd/install.go b/cmd/install.go index ddfbf370b4..d826270d99 100644 --- a/cmd/install.go +++ b/cmd/install.go @@ -11,9 +11,9 @@ import ( "github.com/spf13/cobra" "github.com/elastic/elastic-package/internal/cobraext" - "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/packages/installer" + "github.com/elastic/elastic-package/internal/stack" ) const installLongDescription = `Use this command to install the package in Kibana. @@ -49,7 +49,7 @@ func installCommandAction(cmd *cobra.Command, _ []string) error { return cobraext.FlagParsingError(err, cobraext.BuildSkipValidationFlagName) } - kibanaClient, err := kibana.NewClient() + kibanaClient, err := stack.NewKibanaClient() if err != nil { return fmt.Errorf("could not create kibana client: %w", err) } diff --git a/cmd/uninstall.go b/cmd/uninstall.go index 4d3a5b0d59..b9736931ff 100644 --- a/cmd/uninstall.go +++ b/cmd/uninstall.go @@ -11,9 +11,9 @@ import ( "github.com/spf13/cobra" "github.com/elastic/elastic-package/internal/cobraext" - "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/packages/installer" + "github.com/elastic/elastic-package/internal/stack" ) const uninstallLongDescription = `Use this command to uninstall the package in Kibana. @@ -40,7 +40,7 @@ func uninstallCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("locating package root failed: %w", err) } - kibanaClient, err := kibana.NewClient() + kibanaClient, err := stack.NewKibanaClient() if err != nil { return fmt.Errorf("could not create kibana client: %w", err) } diff --git a/internal/dump/agentpolicies_test.go b/internal/dump/agentpolicies_test.go index d60f105abe..681fb74a6a 100644 --- a/internal/dump/agentpolicies_test.go +++ b/internal/dump/agentpolicies_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/elastic/elastic-package/internal/kibana" + "github.com/elastic/elastic-package/internal/stack" ) func TestDumpAgentPolicies(t *testing.T) { @@ -72,7 +73,7 @@ type agentPoliciesDumpSuite struct { func (s *agentPoliciesDumpSuite) SetupTest() { _, err := os.Stat(s.DumpDirAll) if errors.Is(err, os.ErrNotExist) { - client, err := kibana.NewClient() + client, err := stack.NewKibanaClient() s.Require().NoError(err) dumper := NewAgentPoliciesDumper(client) @@ -85,7 +86,7 @@ func (s *agentPoliciesDumpSuite) SetupTest() { _, err = os.Stat(s.DumpDirPackage) if errors.Is(err, os.ErrNotExist) { - client, err := kibana.NewClient() + client, err := stack.NewKibanaClient() s.Require().NoError(err) dumper := NewAgentPoliciesDumper(client) @@ -98,7 +99,7 @@ func (s *agentPoliciesDumpSuite) SetupTest() { _, err = os.Stat(s.DumpDirAgentPolicy) if errors.Is(err, os.ErrNotExist) { - client, err := kibana.NewClient() + client, err := stack.NewKibanaClient() s.Require().NoError(err) dumper := NewAgentPoliciesDumper(client) diff --git a/internal/kibana/client.go b/internal/kibana/client.go index f47d9d599d..beb292a332 100644 --- a/internal/kibana/client.go +++ b/internal/kibana/client.go @@ -7,18 +7,19 @@ package kibana import ( "bytes" "crypto/tls" + "errors" "fmt" "io" "net/http" "net/url" - "os" "github.com/elastic/elastic-package/internal/certs" "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/logger" - "github.com/elastic/elastic-package/internal/stack" ) +var ErrUndefinedHost = errors.New("missing kibana host") + // Client is responsible for exporting dashboards from Kibana. type Client struct { host string @@ -34,24 +35,13 @@ type ClientOption func(*Client) // NewClient creates a new instance of the client. func NewClient(opts ...ClientOption) (*Client, error) { - host := os.Getenv(stack.KibanaHostEnv) - username := os.Getenv(stack.ElasticsearchUsernameEnv) - password := os.Getenv(stack.ElasticsearchPasswordEnv) - certificateAuthority := os.Getenv(stack.CACertificateEnv) - - c := &Client{ - host: host, - username: username, - password: password, - certificateAuthority: certificateAuthority, - } - + c := &Client{} for _, opt := range opts { opt(c) } if c.host == "" { - return nil, stack.UndefinedEnvError(stack.KibanaHostEnv) + return nil, ErrUndefinedHost } return c, nil @@ -71,6 +61,20 @@ func TLSSkipVerify() ClientOption { } } +// Username option sets the username to be used by the client. +func Username(username string) ClientOption { + return func(c *Client) { + c.username = username + } +} + +// Password option sets the password to be used by the client. +func Password(password string) ClientOption { + return func(c *Client) { + c.password = password + } +} + // CertificateAuthority sets the certificate authority to be used by the client. func CertificateAuthority(certificateAuthority string) ClientOption { return func(c *Client) { diff --git a/internal/kibana/status.go b/internal/kibana/status.go index d13179f749..052c3a8c39 100644 --- a/internal/kibana/status.go +++ b/internal/kibana/status.go @@ -30,6 +30,11 @@ func (v VersionInfo) IsSnapshot() bool { type statusType struct { Version VersionInfo `json:"version"` + Status struct { + Overall struct { + Level string `json:"level"` + } `json:"overall"` + } `json:"status"` } // Version method returns the version of Kibana (Elastic stack) @@ -52,3 +57,26 @@ func (c *Client) Version() (VersionInfo, error) { return status.Version, nil } + +// CheckHealth returns the status of Kibana (Elastic stack) +func (c *Client) CheckHealth() error { + statusCode, respBody, err := c.get(StatusAPI) + if err != nil { + return fmt.Errorf("could not reach status endpoint: %w", err) + } + + if statusCode != http.StatusOK { + return fmt.Errorf("could not get status data; API status code = %d; response body = %s", statusCode, respBody) + } + + var status statusType + err = json.Unmarshal(respBody, &status) + if err != nil { + return fmt.Errorf("unmarshalling response failed (body: \n%s): %w", respBody, err) + } + + if status.Status.Overall.Level != "available" { + return fmt.Errorf("kibana in unhealthy state: %s", status.Status.Overall.Level) + } + return nil +} diff --git a/internal/stack/clients.go b/internal/stack/clients.go index 0cadcb0b6a..20a185c01f 100644 --- a/internal/stack/clients.go +++ b/internal/stack/clients.go @@ -5,6 +5,7 @@ import ( "os" "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/kibana" ) func NewElasticsearchClient(customOptions ...elasticsearch.ClientOption) (*elasticsearch.Client, error) { @@ -24,3 +25,20 @@ func NewElasticsearchClient(customOptions ...elasticsearch.ClientOption) (*elast return client, err } + +func NewKibanaClient(customOptions ...kibana.ClientOption) (*kibana.Client, error) { + options := []kibana.ClientOption{ + kibana.Address(os.Getenv(ElasticsearchHostEnv)), + kibana.Password(os.Getenv(ElasticsearchPasswordEnv)), + kibana.Username(os.Getenv(ElasticsearchUsernameEnv)), + kibana.CertificateAuthority(os.Getenv(CACertificateEnv)), + } + options = append(options, customOptions...) + client, err := kibana.NewClient(options...) + + if errors.Is(err, kibana.ErrUndefinedHost) { + return nil, UndefinedEnvError(ElasticsearchHostEnv) + } + + return client, err +} diff --git a/internal/testrunner/runners/asset/runner.go b/internal/testrunner/runners/asset/runner.go index bbb997d161..e218410a2b 100644 --- a/internal/testrunner/runners/asset/runner.go +++ b/internal/testrunner/runners/asset/runner.go @@ -8,10 +8,10 @@ import ( "fmt" "strings" - "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/packages/installer" + "github.com/elastic/elastic-package/internal/stack" "github.com/elastic/elastic-package/internal/testrunner" ) @@ -76,7 +76,7 @@ func (r *runner) run() ([]testrunner.TestResult, error) { } logger.Debug("installing package...") - kibanaClient, err := kibana.NewClient() + kibanaClient, err := stack.NewKibanaClient() if err != nil { return result.WithError(fmt.Errorf("could not create kibana client: %w", err)) } diff --git a/internal/testrunner/runners/system/servicedeployer/custom_agent.go b/internal/testrunner/runners/system/servicedeployer/custom_agent.go index 42563745b9..41841c672a 100644 --- a/internal/testrunner/runners/system/servicedeployer/custom_agent.go +++ b/internal/testrunner/runners/system/servicedeployer/custom_agent.go @@ -15,7 +15,6 @@ import ( "github.com/elastic/elastic-package/internal/docker" "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/install" - "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" "github.com/elastic/elastic-package/internal/stack" @@ -54,7 +53,7 @@ func (d *CustomAgentDeployer) SetUp(inCtxt ServiceContext) (DeployedService, err return nil, fmt.Errorf("can't read application configuration: %w", err) } - kibanaClient, err := kibana.NewClient() + kibanaClient, err := stack.NewKibanaClient() if err != nil { return nil, fmt.Errorf("can't create Kibana client: %w", err) } diff --git a/internal/testrunner/runners/system/servicedeployer/kubernetes.go b/internal/testrunner/runners/system/servicedeployer/kubernetes.go index c2fe36d9e2..0c20bd9c32 100644 --- a/internal/testrunner/runners/system/servicedeployer/kubernetes.go +++ b/internal/testrunner/runners/system/servicedeployer/kubernetes.go @@ -16,7 +16,6 @@ import ( "text/template" "github.com/elastic/elastic-package/internal/install" - "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/kind" "github.com/elastic/elastic-package/internal/kubectl" "github.com/elastic/elastic-package/internal/logger" @@ -149,7 +148,7 @@ func findKubernetesDefinitions(definitionsDir string) ([]string, error) { func installElasticAgentInCluster() error { logger.Debug("install Elastic Agent in the Kubernetes cluster") - kibanaClient, err := kibana.NewClient() + kibanaClient, err := stack.NewKibanaClient() if err != nil { return fmt.Errorf("can't create Kibana client: %w", err) } From 3084d0a69f079324c0ea1ade9831d7d362a3563d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 21 Aug 2023 16:59:45 +0200 Subject: [PATCH 11/62] Use kibana client from stack in serverless --- internal/serverless/project.go | 62 +++++++++++++++------------------- internal/stack/serverless.go | 10 ++++++ 2 files changed, 37 insertions(+), 35 deletions(-) diff --git a/internal/serverless/project.go b/internal/serverless/project.go index 0111e322d6..ad2dbd1fa0 100644 --- a/internal/serverless/project.go +++ b/internal/serverless/project.go @@ -13,6 +13,7 @@ import ( "time" "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/logger" ) @@ -40,6 +41,7 @@ type Project struct { } `json:"endpoints"` ElasticsearchClient *elasticsearch.Client + KibanaClient *kibana.Client } type serviceHealthy func(context.Context, *Project) error @@ -48,7 +50,7 @@ func (p *Project) EnsureHealthy(ctx context.Context) error { if err := p.ensureElasticserchHealthy(ctx); err != nil { return fmt.Errorf("elasticsearch not healthy: %w", err) } - if err := p.ensureServiceHealthy(ctx, getKibanaHealthy); err != nil { + if err := p.ensureKibanaHealthy(ctx); err != nil { return fmt.Errorf("kibana not healthy: %w", err) } if err := p.ensureServiceHealthy(ctx, getFleetHealthy); err != nil { @@ -68,7 +70,7 @@ func (p *Project) Status(ctx context.Context) (map[string]string, error) { status = map[string]string{ "elasticsearch": healthStatus(p.getESHealth(ctx)), - "kibana": healthStatus(getKibanaHealthy(ctx, p)), + "kibana": healthStatus(p.getKibanaHealth()), "fleet": healthStatus(getFleetHealthy(ctx, p)), } return status, nil @@ -95,6 +97,27 @@ func (p *Project) ensureElasticserchHealthy(ctx context.Context) error { return nil } +func (p *Project) ensureKibanaHealthy(ctx context.Context) error { + timer := time.NewTimer(time.Millisecond) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } + + err := p.KibanaClient.CheckHealth() + if err != nil { + logger.Debugf("service not ready: %s", err.Error()) + timer.Reset(time.Second * 5) + continue + } + + return nil + } + return nil +} + func (p *Project) ensureServiceHealthy(ctx context.Context, serviceFunc serviceHealthy) error { timer := time.NewTimer(time.Millisecond) for { @@ -158,39 +181,8 @@ func (p *Project) getESHealth(ctx context.Context) error { return p.ElasticsearchClient.CheckHealth(ctx) } -func getKibanaHealthy(ctx context.Context, project *Project) error { - client, err := NewClient( - WithAddress(project.Endpoints.Kibana), - WithUsername(project.Credentials.Username), - WithPassword(project.Credentials.Password), - ) - if err != nil { - return err - } - - statusCode, respBody, err := client.get(ctx, "/api/status") - if err != nil { - return fmt.Errorf("failed to query kibana status: %w", err) - } - if statusCode != http.StatusOK { - return fmt.Errorf("unexpected status code %d, body: %s", statusCode, string(respBody)) - } - - var status struct { - Status struct { - Overall struct { - Level string `json:"level"` - } `json:"overall"` - } `json:"status"` - } - if err := json.Unmarshal(respBody, &status); err != nil { - logger.Debugf("Unable to decode response: %v body: %s", err, string(respBody)) - return err - } - if status.Status.Overall.Level == "available" { - return nil - } - return fmt.Errorf("kibana unhealthy: %s", status.Status.Overall.Level) +func (p *Project) getKibanaHealth() error { + return p.KibanaClient.CheckHealth() } func getFleetHealthy(ctx context.Context, project *Project) error { diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index eab50c4753..4f945b53c3 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -18,6 +18,7 @@ import ( "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/docker" "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" "github.com/elastic/elastic-package/internal/serverless" @@ -127,6 +128,15 @@ func (sp *serverlessProvider) createClients(project *serverless.Project) (*serve return project, fmt.Errorf("failed to create elasticsearch client") } + project.KibanaClient, err = NewKibanaClient( + kibana.Address(project.Endpoints.Kibana), + kibana.Username(project.Credentials.Username), + kibana.Password(project.Credentials.Password), + ) + if err != nil { + return project, fmt.Errorf("failed to create kibana client") + } + return project, nil } From 27fff3cb89ee7239079afcadd5333bc16e975e0a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 21 Aug 2023 17:13:03 +0200 Subject: [PATCH 12/62] Fix kibana client --- internal/stack/clients.go | 2 +- internal/testrunner/runners/system/runner.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/stack/clients.go b/internal/stack/clients.go index 20a185c01f..4b31d2cb84 100644 --- a/internal/stack/clients.go +++ b/internal/stack/clients.go @@ -28,7 +28,7 @@ func NewElasticsearchClient(customOptions ...elasticsearch.ClientOption) (*elast func NewKibanaClient(customOptions ...kibana.ClientOption) (*kibana.Client, error) { options := []kibana.ClientOption{ - kibana.Address(os.Getenv(ElasticsearchHostEnv)), + kibana.Address(os.Getenv(KibanaHostEnv)), kibana.Password(os.Getenv(ElasticsearchPasswordEnv)), kibana.Username(os.Getenv(ElasticsearchUsernameEnv)), kibana.CertificateAuthority(os.Getenv(CACertificateEnv)), diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 31ecbc55e3..6d651a071f 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -476,7 +476,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext return result.WithError(fmt.Errorf("unable to reload system test case configuration: %w", err)) } - kib, err := kibana.NewClient() + kib, err := stack.NewKibanaClient() if err != nil { return result.WithError(fmt.Errorf("can't create Kibana client: %w", err)) } From 57eb28e06fd12872c33d1e473072a36cfd1b2ad1 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 21 Aug 2023 17:52:08 +0200 Subject: [PATCH 13/62] Use kibana stack client to create agent and package policy --- internal/serverless/client.go | 3 + internal/stack/clients.go | 4 ++ internal/stack/serverless.go | 82 +++++++-------------------- internal/stack/serverlessresources.go | 3 +- 4 files changed, 29 insertions(+), 63 deletions(-) diff --git a/internal/serverless/client.go b/internal/serverless/client.go index 181843215a..456cfd53a6 100644 --- a/internal/serverless/client.go +++ b/internal/serverless/client.go @@ -184,6 +184,9 @@ func (c *Client) CreateProject(name, region, project string) (*Project, error) { serverlessProject := &Project{url: c.host, apiKey: c.apiKey} err = json.Unmarshal(respBody, &serverlessProject) + if err != nil { + return nil, fmt.Errorf("error while decoding create project response: %w", err) + } serverlessProject, err = c.ResetCredentials(ctx, serverlessProject) if err != nil { diff --git a/internal/stack/clients.go b/internal/stack/clients.go index 4b31d2cb84..0756c8b781 100644 --- a/internal/stack/clients.go +++ b/internal/stack/clients.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package stack import ( diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 4f945b53c3..c387eae274 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -296,79 +296,37 @@ func (sp *serverlessProvider) startLocalAgent(options Options, config Config) er return nil } -const serverlessKibanaAgentPolicy = `{ - "name": "Elastic-Agent (elastic-package)", - "id": "elastic-agent-managed-ep", - "description": "Policy created by elastic-package", - "namespace": "default", - "monitoring_enabled": [ - "logs", - "metrics" - ] -}` - -const serverlessKibanaPackagePolicy = `{ - "name": "system-1", - "policy_id": "elastic-agent-managed-ep", - "package": { - "name": "system", - "version": "%s" - } -}` - -func doKibanaRequest(config Config, req *http.Request) error { - req.SetBasicAuth(config.ElasticsearchUsername, config.ElasticsearchPassword) - req.Header.Add("content-type", "application/json") - req.Header.Add("kbn-xsrf", "elastic-package") - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("performing request failed: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode == http.StatusConflict { - // Already created, go on. - // TODO: We could try to update the policy. - return nil - } - if resp.StatusCode >= 300 { - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("request failed with status %v and could not read body: %w", resp.StatusCode, err) - } - return fmt.Errorf("request failed with status %v and response %v", resp.StatusCode, string(body)) - } - return nil -} - func (sp *serverlessProvider) createAgentPolicy(config Config, stackVersion string) error { - agentPoliciesURL, err := url.JoinPath(config.KibanaHost, "/api/fleet/agent_policies") + systemVersion, err := getPackageVersion("https://epr.elastic.co", "system", stackVersion) if err != nil { - return fmt.Errorf("failed to build url for agent policies: %w", err) + return fmt.Errorf("could not get the system package version for kibana %v: %w", stackVersion, err) } - req, err := http.NewRequest(http.MethodPost, agentPoliciesURL, strings.NewReader(serverlessKibanaAgentPolicy)) + + client, err := NewKibanaClient() if err != nil { - return fmt.Errorf("failed to initialize request to create agent policy: %w", err) + return fmt.Errorf("failed to create kibana client: %w", err) + } + + policy := kibana.Policy{ + ID: "elastic-agent-managed-ep", + Name: "Elastic-Agent (elastic-package)", + Description: "Policy created by elastic-package", + Namespace: "default", + MonitoringEnabled: []string{"logs", "metrics"}, } - err = doKibanaRequest(config, req) + newPolicy, err := client.CreatePolicy(policy) if err != nil { return fmt.Errorf("error while creating agent policy: %w", err) } - systemVersion, err := getPackageVersion("https://epr.elastic.co", "system", stackVersion) - if err != nil { - return fmt.Errorf("could not get the system package version for kibana %v: %w", stackVersion, err) + packagePolicy := kibana.PackagePolicy{ + Name: "system-1", + PolicyID: newPolicy.ID, } + packagePolicy.Package.Name = "system" + packagePolicy.Package.Version = systemVersion - packagePoliciesURL, err := url.JoinPath(config.KibanaHost, "/api/fleet/package_policies") - if err != nil { - return fmt.Errorf("failed to build url for package policies: %w", err) - } - packagePolicy := fmt.Sprintf(serverlessKibanaPackagePolicy, systemVersion) - req, err = http.NewRequest(http.MethodPost, packagePoliciesURL, strings.NewReader(packagePolicy)) - if err != nil { - return fmt.Errorf("failed to initialize request to create package policy: %w", err) - } - err = doKibanaRequest(config, req) + _, err = client.CreatePackagePolicy(packagePolicy) if err != nil { return fmt.Errorf("error while creating package policy: %w", err) } diff --git a/internal/stack/serverlessresources.go b/internal/stack/serverlessresources.go index 8ce61be923..e29e616c0d 100644 --- a/internal/stack/serverlessresources.go +++ b/internal/stack/serverlessresources.go @@ -10,9 +10,10 @@ import ( "path/filepath" "strings" + "github.com/elastic/go-resource" + "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/profile" - "github.com/elastic/go-resource" ) const ( From 81b213c367004ab4adfa18687407b915e2337906 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 21 Aug 2023 18:38:18 +0200 Subject: [PATCH 14/62] Add method to get default fleet URL to kibana client --- internal/kibana/fleet.go | 44 ++++++++++++++++++++++++++++++++++ internal/serverless/project.go | 35 ++------------------------- 2 files changed, 46 insertions(+), 33 deletions(-) create mode 100644 internal/kibana/fleet.go diff --git a/internal/kibana/fleet.go b/internal/kibana/fleet.go new file mode 100644 index 0000000000..a55d41eda8 --- /dev/null +++ b/internal/kibana/fleet.go @@ -0,0 +1,44 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package kibana + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" +) + +func (c *Client) DefaultFleetServerURL() (string, error) { + path := fmt.Sprintf("%s/fleet_server_hosts", FleetAPI) + + statusCode, respBody, err := c.get(path) + if err != nil { + return "", fmt.Errorf("could not reach fleet server hosts endpoint: %w", err) + } + + if statusCode != http.StatusOK { + return "", fmt.Errorf("could not get status data; API status code = %d; response body = %s", statusCode, respBody) + } + + var hosts struct { + Items []struct { + IsDefault bool `json:"is_default"` + HostURLs []string `json:"host_urls"` + } `json:"items"` + } + err = json.Unmarshal(respBody, &hosts) + if err != nil { + return "", fmt.Errorf("failed to decode response: %w", err) + } + + for _, server := range hosts.Items { + if server.IsDefault && len(server.HostURLs) > 0 { + return server.HostURLs[0], nil + } + } + + return "", errors.New("could not find the fleet server URL for this project") +} diff --git a/internal/serverless/project.go b/internal/serverless/project.go index ad2dbd1fa0..490e60ba63 100644 --- a/internal/serverless/project.go +++ b/internal/serverless/project.go @@ -6,8 +6,6 @@ package serverless import ( "context" - "encoding/json" - "errors" "fmt" "net/http" "time" @@ -140,41 +138,12 @@ func (p *Project) ensureServiceHealthy(ctx context.Context, serviceFunc serviceH } func (p *Project) DefaultFleetServerURL(ctx context.Context) (string, error) { - client, err := NewClient( - WithAddress(p.Endpoints.Kibana), - WithUsername(p.Credentials.Username), - WithPassword(p.Credentials.Password), - ) - if err != nil { - return "", err - } - statusCode, respBody, err := client.get(ctx, "/api/fleet/fleet_server_hosts") + fleetURL, err := p.KibanaClient.DefaultFleetServerURL() if err != nil { return "", fmt.Errorf("failed to query fleet server hosts: %w", err) } - if statusCode != http.StatusOK { - return "", fmt.Errorf("unexpected status code %d, body: %s", statusCode, string(respBody)) - } - - var hosts struct { - Items []struct { - IsDefault bool `json:"is_default"` - HostURLs []string `json:"host_urls"` - } `json:"items"` - } - err = json.Unmarshal(respBody, &hosts) - if err != nil { - return "", fmt.Errorf("failed to decode response: %w", err) - } - - for _, server := range hosts.Items { - if server.IsDefault && len(server.HostURLs) > 0 { - return server.HostURLs[0], nil - } - } - - return "", errors.New("could not find the fleet server URL for this project") + return fleetURL, nil } func (p *Project) getESHealth(ctx context.Context) error { From 4e800e3f2614fe6333a186f13d2817818b0da528 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 22 Aug 2023 12:46:11 +0200 Subject: [PATCH 15/62] Move creation of agent policy to project --- internal/serverless/project.go | 73 +++++++++++++++++++++++ internal/stack/serverless.go | 104 +++++---------------------------- 2 files changed, 89 insertions(+), 88 deletions(-) diff --git a/internal/serverless/project.go b/internal/serverless/project.go index 490e60ba63..38ac827163 100644 --- a/internal/serverless/project.go +++ b/internal/serverless/project.go @@ -6,8 +6,11 @@ package serverless import ( "context" + "encoding/json" "fmt" + "io" "net/http" + "net/url" "time" "github.com/elastic/elastic-package/internal/elasticsearch" @@ -174,3 +177,73 @@ func getFleetHealthy(ctx context.Context, project *Project) error { return nil } + +func (p *Project) CreateAgentPolicy(stackVersion string) error { + systemVersion, err := getPackageVersion("https://epr.elastic.co", "system", stackVersion) + if err != nil { + return fmt.Errorf("could not get the system package version for kibana %v: %w", stackVersion, err) + } + + policy := kibana.Policy{ + ID: "elastic-agent-managed-ep", + Name: "Elastic-Agent (elastic-package)", + Description: "Policy created by elastic-package", + Namespace: "default", + MonitoringEnabled: []string{"logs", "metrics"}, + } + newPolicy, err := p.KibanaClient.CreatePolicy(policy) + if err != nil { + return fmt.Errorf("error while creating agent policy: %w", err) + } + + packagePolicy := kibana.PackagePolicy{ + Name: "system-1", + PolicyID: newPolicy.ID, + Namespace: newPolicy.Namespace, + } + packagePolicy.Package.Name = "system" + packagePolicy.Package.Version = systemVersion + + _, err = p.KibanaClient.CreatePackagePolicy(packagePolicy) + if err != nil { + return fmt.Errorf("error while creating package policy: %w", err) + } + + return nil +} + +func getPackageVersion(registryURL, packageName, stackVersion string) (string, error) { + searchURL, err := url.JoinPath(registryURL, "search") + if err != nil { + return "", fmt.Errorf("could not build URL: %w", err) + } + searchURL = fmt.Sprintf("%s?package=%s&kibana.version=%s", searchURL, packageName, stackVersion) + resp, err := http.Get(searchURL) + if err != nil { + return "", fmt.Errorf("request failed (url: %s): %w", searchURL, err) + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + return "", fmt.Errorf("unexpected status code %v", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read response body: %w", err) + } + var packages []struct { + Name string `json:"name"` + Version string `json:"version"` + } + err = json.Unmarshal(body, &packages) + if err != nil { + return "", fmt.Errorf("failed to parse response body: %w", err) + } + if len(packages) != 1 { + return "", fmt.Errorf("expected 1 package, obtained %v", len(packages)) + } + if found := packages[0].Name; found != packageName { + return "", fmt.Errorf("expected package %s, found %s", packageName, found) + } + + return packages[0].Version, nil +} diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index c387eae274..816f63b691 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -6,12 +6,8 @@ package stack import ( "context" - "encoding/json" "errors" "fmt" - "io" - "net/http" - "net/url" "strings" "time" @@ -53,11 +49,12 @@ type projectSettings struct { StackVersion string } -func (sp *serverlessProvider) createProject(settings projectSettings, options Options) (Config, error) { +func (sp *serverlessProvider) createProject(settings projectSettings, options Options, conf Config) (Config, error) { project, err := sp.client.CreateProject(settings.Name, settings.Region, settings.Type) if err != nil { return Config{}, fmt.Errorf("failed to create %s project %s in %s: %w", settings.Type, settings.Name, settings.Region, err) } + // project, _ := sp.currentProject(conf) ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30) defer cancel() @@ -166,8 +163,13 @@ func (sp *serverlessProvider) currentProject(config Config) (*serverless.Project project.Credentials.Username = config.ElasticsearchUsername project.Credentials.Password = config.ElasticsearchPassword - fleetURL, ok := config.Parameters[paramServerlessFleetURL] - if !ok { + project, err = sp.createClients(project) + if err != nil { + return nil, fmt.Errorf("failed to create project client") + } + + fleetURL := config.Parameters[paramServerlessFleetURL] + if true { fleetURL, err = project.DefaultFleetServerURL(context.Background()) if err != nil { return nil, fmt.Errorf("failed to get fleet URL: %w", err) @@ -175,11 +177,6 @@ func (sp *serverlessProvider) currentProject(config Config) (*serverless.Project } project.Endpoints.Fleet = fleetURL - project, err = sp.createClients(project) - if err != nil { - return nil, fmt.Errorf("failed to create project client") - } - return project, nil } @@ -228,13 +225,18 @@ func (sp *serverlessProvider) BootUp(options Options) error { return err case errProjectNotExist: logger.Infof("Creating project %q", settings.Name) - config, err = sp.createProject(settings, options) + config, err = sp.createProject(settings, options, config) if err != nil { return fmt.Errorf("failed to create deployment: %w", err) } + project, err = sp.currentProject(config) + if err != nil { + return fmt.Errorf("failed to retrieve latest project created: %w", err) + } + logger.Infof("Creating agent policy") - err = sp.createAgentPolicy(config, options.StackVersion) + err = project.CreateAgentPolicy(options.StackVersion) if err != nil { return fmt.Errorf("failed to create agent policy: %w", err) } @@ -296,80 +298,6 @@ func (sp *serverlessProvider) startLocalAgent(options Options, config Config) er return nil } -func (sp *serverlessProvider) createAgentPolicy(config Config, stackVersion string) error { - systemVersion, err := getPackageVersion("https://epr.elastic.co", "system", stackVersion) - if err != nil { - return fmt.Errorf("could not get the system package version for kibana %v: %w", stackVersion, err) - } - - client, err := NewKibanaClient() - if err != nil { - return fmt.Errorf("failed to create kibana client: %w", err) - } - - policy := kibana.Policy{ - ID: "elastic-agent-managed-ep", - Name: "Elastic-Agent (elastic-package)", - Description: "Policy created by elastic-package", - Namespace: "default", - MonitoringEnabled: []string{"logs", "metrics"}, - } - newPolicy, err := client.CreatePolicy(policy) - if err != nil { - return fmt.Errorf("error while creating agent policy: %w", err) - } - - packagePolicy := kibana.PackagePolicy{ - Name: "system-1", - PolicyID: newPolicy.ID, - } - packagePolicy.Package.Name = "system" - packagePolicy.Package.Version = systemVersion - - _, err = client.CreatePackagePolicy(packagePolicy) - if err != nil { - return fmt.Errorf("error while creating package policy: %w", err) - } - - return nil -} - -func getPackageVersion(registryURL, packageName, stackVersion string) (string, error) { - searchURL, err := url.JoinPath(registryURL, "search") - if err != nil { - return "", fmt.Errorf("could not build URL: %w", err) - } - searchURL = fmt.Sprintf("%s?package=%s&kibana.version=%s", searchURL, packageName, stackVersion) - resp, err := http.Get(searchURL) - if err != nil { - return "", fmt.Errorf("request failed (url: %s): %w", searchURL, err) - } - defer resp.Body.Close() - if resp.StatusCode >= 300 { - return "", fmt.Errorf("unexpected status code %v", resp.StatusCode) - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("failed to read response body: %w", err) - } - var packages []struct { - Name string `json:"name"` - Version string `json:"version"` - } - err = json.Unmarshal(body, &packages) - if err != nil { - return "", fmt.Errorf("failed to parse response body: %w", err) - } - if len(packages) != 1 { - return "", fmt.Errorf("expected 1 package, obtained %v", len(packages)) - } - if found := packages[0].Name; found != packageName { - return "", fmt.Errorf("expected package %s, found %s", packageName, found) - } - - return packages[0].Version, nil -} - func (sp *serverlessProvider) TearDown(options Options) error { config, err := LoadConfig(sp.profile) if err != nil { From 3302ca2bab6289f6ff285d5063cdd7fa11995370 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 22 Aug 2023 12:51:37 +0200 Subject: [PATCH 16/62] Fleet should be just checked if project is observability --- internal/serverless/project.go | 4 +++- internal/stack/serverless.go | 14 ++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/internal/serverless/project.go b/internal/serverless/project.go index 38ac827163..c63107708c 100644 --- a/internal/serverless/project.go +++ b/internal/serverless/project.go @@ -72,7 +72,9 @@ func (p *Project) Status(ctx context.Context) (map[string]string, error) { status = map[string]string{ "elasticsearch": healthStatus(p.getESHealth(ctx)), "kibana": healthStatus(p.getKibanaHealth()), - "fleet": healthStatus(getFleetHealthy(ctx, p)), + } + if p.Type == "observability" { + status["fleet"] = healthStatus(getFleetHealthy(ctx, p)) } return status, nil } diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 816f63b691..db6ef43aae 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -92,11 +92,13 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op return Config{}, fmt.Errorf("failed to create project client") } - config.Parameters[paramServerlessFleetURL], err = project.DefaultFleetServerURL(ctx) - if err != nil { - return Config{}, fmt.Errorf("failed to get fleet URL: %w", err) + if project.Type == "observability" { + config.Parameters[paramServerlessFleetURL], err = project.DefaultFleetServerURL(ctx) + if err != nil { + return Config{}, fmt.Errorf("failed to get fleet URL: %w", err) + } + project.Endpoints.Fleet = config.Parameters[paramServerlessFleetURL] } - project.Endpoints.Fleet = config.Parameters[paramServerlessFleetURL] printUserConfig(options.Printer, config) @@ -168,6 +170,10 @@ func (sp *serverlessProvider) currentProject(config Config) (*serverless.Project return nil, fmt.Errorf("failed to create project client") } + if project.Type != "observability" { + return project, nil + } + fleetURL := config.Parameters[paramServerlessFleetURL] if true { fleetURL, err = project.DefaultFleetServerURL(context.Background()) From 54395dca333412b4236e9aebd7f23d665f1cf183 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 22 Aug 2023 13:09:52 +0200 Subject: [PATCH 17/62] Use raw http request --- internal/serverless/project.go | 40 ++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/internal/serverless/project.go b/internal/serverless/project.go index c63107708c..05253d048b 100644 --- a/internal/serverless/project.go +++ b/internal/serverless/project.go @@ -18,6 +18,8 @@ import ( "github.com/elastic/elastic-package/internal/logger" ) +const EPR_URL = "https://epr.elastic.co" + // Project represents a serverless project type Project struct { url string @@ -160,28 +162,41 @@ func (p *Project) getKibanaHealth() error { } func getFleetHealthy(ctx context.Context, project *Project) error { - client, err := NewClient( - WithAddress(project.Endpoints.Fleet), - WithUsername(project.Credentials.Username), - WithPassword(project.Credentials.Password), - ) + statusURL, err := url.JoinPath(project.Endpoints.Fleet, "api/status") if err != nil { - return err + return fmt.Errorf("could not build URL: %w", err) } - - statusCode, respBody, err := client.get(ctx, "/api/status") + logger.Debugf("GET %s", statusURL) + resp, err := http.Get(statusURL) + if err != nil { + return fmt.Errorf("request failed (url: %s): %w", statusURL, err) + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + return fmt.Errorf("unexpected status code %v", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) if err != nil { - return fmt.Errorf("failed to query fleet status: %w", err) + return fmt.Errorf("failed to read response body: %w", err) + } + var status struct { + Name string `json:"name"` + Status string `json:"status"` } - if statusCode != http.StatusOK { - return fmt.Errorf("fleet unhealthy: status code %d, body: %s", statusCode, string(respBody)) + err = json.Unmarshal(body, &status) + if err != nil { + return fmt.Errorf("failed to parse response body: %w", err) } + if status.Status != "HEALTHY" { + return fmt.Errorf("fleet status %s", status.Status) + + } return nil } func (p *Project) CreateAgentPolicy(stackVersion string) error { - systemVersion, err := getPackageVersion("https://epr.elastic.co", "system", stackVersion) + systemVersion, err := getPackageVersion(EPR_URL, "system", stackVersion) if err != nil { return fmt.Errorf("could not get the system package version for kibana %v: %w", stackVersion, err) } @@ -220,6 +235,7 @@ func getPackageVersion(registryURL, packageName, stackVersion string) (string, e return "", fmt.Errorf("could not build URL: %w", err) } searchURL = fmt.Sprintf("%s?package=%s&kibana.version=%s", searchURL, packageName, stackVersion) + logger.Debugf("GET %s", searchURL) resp, err := http.Get(searchURL) if err != nil { return "", fmt.Errorf("request failed (url: %s): %w", searchURL, err) From bf42d9f4222437e35e4dac57d02b51283e6563ce Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 22 Aug 2023 16:43:18 +0200 Subject: [PATCH 18/62] Use http client for Fleet Raise an error in case elasticsearch project is created, since it does not have fleet service. --- internal/serverless/project.go | 28 +++++++++++++++------------- internal/stack/serverless.go | 26 ++++++++++++-------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/internal/serverless/project.go b/internal/serverless/project.go index 05253d048b..4b3f7f9fdd 100644 --- a/internal/serverless/project.go +++ b/internal/serverless/project.go @@ -56,7 +56,7 @@ func (p *Project) EnsureHealthy(ctx context.Context) error { if err := p.ensureKibanaHealthy(ctx); err != nil { return fmt.Errorf("kibana not healthy: %w", err) } - if err := p.ensureServiceHealthy(ctx, getFleetHealthy); err != nil { + if err := p.ensureFleetHealthy(ctx); err != nil { return fmt.Errorf("fleet not healthy: %w", err) } return nil @@ -74,9 +74,7 @@ func (p *Project) Status(ctx context.Context) (map[string]string, error) { status = map[string]string{ "elasticsearch": healthStatus(p.getESHealth(ctx)), "kibana": healthStatus(p.getKibanaHealth()), - } - if p.Type == "observability" { - status["fleet"] = healthStatus(getFleetHealthy(ctx, p)) + "fleet": healthStatus(p.getFleetHealth(ctx)), } return status, nil } @@ -92,7 +90,7 @@ func (p *Project) ensureElasticserchHealthy(ctx context.Context) error { err := p.ElasticsearchClient.CheckHealth(ctx) if err != nil { - logger.Debugf("service not ready: %s", err.Error()) + logger.Debugf("Elasticsearch service not ready: %s", err.Error()) timer.Reset(time.Second * 5) continue } @@ -113,7 +111,7 @@ func (p *Project) ensureKibanaHealthy(ctx context.Context) error { err := p.KibanaClient.CheckHealth() if err != nil { - logger.Debugf("service not ready: %s", err.Error()) + logger.Debugf("Kibana service not ready: %s", err.Error()) timer.Reset(time.Second * 5) continue } @@ -123,7 +121,7 @@ func (p *Project) ensureKibanaHealthy(ctx context.Context) error { return nil } -func (p *Project) ensureServiceHealthy(ctx context.Context, serviceFunc serviceHealthy) error { +func (p *Project) ensureFleetHealthy(ctx context.Context) error { timer := time.NewTimer(time.Millisecond) for { select { @@ -132,9 +130,9 @@ func (p *Project) ensureServiceHealthy(ctx context.Context, serviceFunc serviceH case <-timer.C: } - err := serviceFunc(ctx, p) + err := p.getFleetHealth(ctx) if err != nil { - logger.Debugf("service not ready: %s", err.Error()) + logger.Debugf("Fleet service not ready: %s", err.Error()) timer.Reset(time.Second * 5) continue } @@ -144,7 +142,7 @@ func (p *Project) ensureServiceHealthy(ctx context.Context, serviceFunc serviceH return nil } -func (p *Project) DefaultFleetServerURL(ctx context.Context) (string, error) { +func (p *Project) DefaultFleetServerURL() (string, error) { fleetURL, err := p.KibanaClient.DefaultFleetServerURL() if err != nil { return "", fmt.Errorf("failed to query fleet server hosts: %w", err) @@ -161,13 +159,17 @@ func (p *Project) getKibanaHealth() error { return p.KibanaClient.CheckHealth() } -func getFleetHealthy(ctx context.Context, project *Project) error { - statusURL, err := url.JoinPath(project.Endpoints.Fleet, "api/status") +func (p *Project) getFleetHealth(ctx context.Context) error { + statusURL, err := url.JoinPath(p.Endpoints.Fleet, "/api/status") if err != nil { return fmt.Errorf("could not build URL: %w", err) } logger.Debugf("GET %s", statusURL) - resp, err := http.Get(statusURL) + req, err := http.NewRequestWithContext(ctx, "GET", statusURL, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) if err != nil { return fmt.Errorf("request failed (url: %s): %w", statusURL, err) } diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index db6ef43aae..957bf09f3f 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -92,13 +92,11 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op return Config{}, fmt.Errorf("failed to create project client") } - if project.Type == "observability" { - config.Parameters[paramServerlessFleetURL], err = project.DefaultFleetServerURL(ctx) - if err != nil { - return Config{}, fmt.Errorf("failed to get fleet URL: %w", err) - } - project.Endpoints.Fleet = config.Parameters[paramServerlessFleetURL] + config.Parameters[paramServerlessFleetURL], err = project.DefaultFleetServerURL() + if err != nil { + return Config{}, fmt.Errorf("failed to get fleet URL: %w", err) } + project.Endpoints.Fleet = config.Parameters[paramServerlessFleetURL] printUserConfig(options.Printer, config) @@ -170,13 +168,9 @@ func (sp *serverlessProvider) currentProject(config Config) (*serverless.Project return nil, fmt.Errorf("failed to create project client") } - if project.Type != "observability" { - return project, nil - } - fleetURL := config.Parameters[paramServerlessFleetURL] if true { - fleetURL, err = project.DefaultFleetServerURL(context.Background()) + fleetURL, err = project.DefaultFleetServerURL() if err != nil { return nil, fmt.Errorf("failed to get fleet URL: %w", err) } @@ -223,6 +217,10 @@ func (sp *serverlessProvider) BootUp(options Options) error { return err } + if settings.Type == "elasticsearch" { + return fmt.Errorf("serverless project type not supported: %s", settings.Type) + } + var project *serverless.Project project, err = sp.currentProject(config) @@ -230,7 +228,7 @@ func (sp *serverlessProvider) BootUp(options Options) error { default: return err case errProjectNotExist: - logger.Infof("Creating project %q", settings.Name) + logger.Infof("Creating %s project: %q", settings.Type, settings.Name) config, err = sp.createProject(settings, options, config) if err != nil { return fmt.Errorf("failed to create deployment: %w", err) @@ -253,9 +251,9 @@ func (sp *serverlessProvider) BootUp(options Options) error { // return fmt.Errorf("failed to replace GeoIP databases: %w", err) // } case nil: - logger.Debugf("Project existed: %s", project.Name) + logger.Debugf("%s project existed: %s", project.Type, project.Name) printUserConfig(options.Printer, config) - logger.Infof("Updating project %s", project.Name) + // logger.Infof("Updating project %s", project.Name) // err = sp.updateDeployment(project, settings) // if err != nil { // return fmt.Errorf("failed to update deployment: %w", err) From 4c448a86dc15e0bd3dbecccf004f494d05af4495 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 22 Aug 2023 17:10:24 +0200 Subject: [PATCH 19/62] Fix lint --- internal/serverless/client.go | 3 --- internal/serverless/project.go | 9 ++------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/internal/serverless/client.go b/internal/serverless/client.go index 456cfd53a6..71a6ee5aa5 100644 --- a/internal/serverless/client.go +++ b/internal/serverless/client.go @@ -193,9 +193,6 @@ func (c *Client) CreateProject(name, region, project string) (*Project, error) { return nil, fmt.Errorf("failed to reset credentials: %w", err) } - bytes, _ := json.MarshalIndent(&serverlessProject, "", " ") - fmt.Printf("Project:\n%s", string(bytes)) - return serverlessProject, err } diff --git a/internal/serverless/project.go b/internal/serverless/project.go index 4b3f7f9fdd..3dc9c35f9e 100644 --- a/internal/serverless/project.go +++ b/internal/serverless/project.go @@ -18,7 +18,7 @@ import ( "github.com/elastic/elastic-package/internal/logger" ) -const EPR_URL = "https://epr.elastic.co" +const eprURL = "https://epr.elastic.co" // Project represents a serverless project type Project struct { @@ -47,8 +47,6 @@ type Project struct { KibanaClient *kibana.Client } -type serviceHealthy func(context.Context, *Project) error - func (p *Project) EnsureHealthy(ctx context.Context) error { if err := p.ensureElasticserchHealthy(ctx); err != nil { return fmt.Errorf("elasticsearch not healthy: %w", err) @@ -97,7 +95,6 @@ func (p *Project) ensureElasticserchHealthy(ctx context.Context) error { return nil } - return nil } func (p *Project) ensureKibanaHealthy(ctx context.Context) error { @@ -118,7 +115,6 @@ func (p *Project) ensureKibanaHealthy(ctx context.Context) error { return nil } - return nil } func (p *Project) ensureFleetHealthy(ctx context.Context) error { @@ -139,7 +135,6 @@ func (p *Project) ensureFleetHealthy(ctx context.Context) error { return nil } - return nil } func (p *Project) DefaultFleetServerURL() (string, error) { @@ -198,7 +193,7 @@ func (p *Project) getFleetHealth(ctx context.Context) error { } func (p *Project) CreateAgentPolicy(stackVersion string) error { - systemVersion, err := getPackageVersion(EPR_URL, "system", stackVersion) + systemVersion, err := getPackageVersion(eprURL, "system", stackVersion) if err != nil { return fmt.Errorf("could not get the system package version for kibana %v: %w", stackVersion, err) } From 8ff731a1b1953e39e938b2ed59a43e13b58b0542 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 23 Aug 2023 11:58:16 +0200 Subject: [PATCH 20/62] Retrieve local services from docker daemon directly --- internal/stack/dump.go | 9 ++++-- internal/stack/serverless.go | 59 +++++++++++++++++++++++++++--------- 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/internal/stack/dump.go b/internal/stack/dump.go index 1e94c1f561..cb53615740 100644 --- a/internal/stack/dump.go +++ b/internal/stack/dump.go @@ -18,8 +18,6 @@ const ( fleetServerService = "fleet-server" ) -var observedServices = []string{"elasticsearch", elasticAgentService, fleetServerService, "kibana", "package-registry"} - // DumpOptions defines dumping options for Elatic stack data. type DumpOptions struct { Output string @@ -50,7 +48,12 @@ func dumpStackLogs(options DumpOptions) error { return fmt.Errorf("can't create output location (path: %s): %w", logsPath, err) } - for _, serviceName := range observedServices { + services, err := localServiceNames(DockerComposeProjectName(options.Profile)) + if err != nil { + return fmt.Errorf("failed to get local services: %w", err) + } + + for _, serviceName := range services { logger.Debugf("Dump stack logs for %s", serviceName) content, err := dockerComposeLogs(serviceName, options.Profile) diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 957bf09f3f..0ede2e5161 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -358,7 +358,7 @@ func (sp *serverlessProvider) Update(options Options) error { } func (sp *serverlessProvider) Dump(options DumpOptions) (string, error) { - return "", fmt.Errorf("not implemented") + return Dump(options) } func (sp *serverlessProvider) Status(options Options) ([]ServiceStatus, error) { @@ -400,32 +400,63 @@ func (sp *serverlessProvider) Status(options Options) ([]ServiceStatus, error) { func (sp *serverlessProvider) localAgentStatus() ([]ServiceStatus, error) { var services []ServiceStatus - // query directly to docker to avoid load environment variables (e.g. STACK_VERSION_VARIANT) and profiles - containerIDs, err := docker.ContainerIDsWithLabel(projectLabelDockerCompose, sp.composeProjectName()) + serviceStatusFunc := func(description docker.ContainerDescription) error { + service, err := newServiceStatus(&description) + if err != nil { + return err + } + services = append(services, *service) + return nil + } + + err := runOnLocalServices(sp.composeProjectName(), serviceStatusFunc) if err != nil { return nil, err } + return services, nil +} + +func localServiceNames(project string) ([]string, error) { + services := []string{} + serviceFunc := func(description docker.ContainerDescription) error { + services = append(services, description.Config.Labels[serviceLabelDockerCompose]) + return nil + } + + err := runOnLocalServices(project, serviceFunc) + if err != nil { + return nil, err + } + + return services, nil +} + +func runOnLocalServices(project string, serviceFunc func(docker.ContainerDescription) error) error { + // query directly to docker to avoid load environment variables (e.g. STACK_VERSION_VARIANT) and profiles + containerIDs, err := docker.ContainerIDsWithLabel(projectLabelDockerCompose, project) + if err != nil { + return err + } + if len(containerIDs) == 0 { - return services, nil + return nil } containerDescriptions, err := docker.InspectContainers(containerIDs...) if err != nil { - return nil, err + return err } for _, containerDescription := range containerDescriptions { - service, err := newServiceStatus(&containerDescription) - if err != nil { - return nil, err - } - if strings.HasSuffix(service.Name, readyServicesSuffix) { + serviceName := containerDescription.Config.Labels[serviceLabelDockerCompose] + if strings.HasSuffix(serviceName, readyServicesSuffix) { continue } - logger.Debugf("Adding Service: \"%v\"", service.Name) - services = append(services, *service) + err := serviceFunc(containerDescription) + if err != nil { + return err + } } - - return services, nil + return nil } From 30055e12716215a69228f710937f251e9449adde Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 23 Aug 2023 13:14:22 +0200 Subject: [PATCH 21/62] Update kibana.CheckHealth comment --- internal/kibana/status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/kibana/status.go b/internal/kibana/status.go index 052c3a8c39..ce365f6097 100644 --- a/internal/kibana/status.go +++ b/internal/kibana/status.go @@ -58,7 +58,7 @@ func (c *Client) Version() (VersionInfo, error) { return status.Version, nil } -// CheckHealth returns the status of Kibana (Elastic stack) +// CheckHealth checks the Kibana health func (c *Client) CheckHealth() error { statusCode, respBody, err := c.get(StatusAPI) if err != nil { From d3df2039cc7dcbd18cb46bab8bfc1784a6557e1e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 23 Aug 2023 13:17:39 +0200 Subject: [PATCH 22/62] Update return in ResetCredentials --- internal/serverless/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/serverless/client.go b/internal/serverless/client.go index 71a6ee5aa5..827e93e5c0 100644 --- a/internal/serverless/client.go +++ b/internal/serverless/client.go @@ -269,7 +269,7 @@ func (c *Client) ResetCredentials(ctx context.Context, project *Project) (*Proje project.Credentials.Username = credentials.Username project.Credentials.Password = credentials.Password - return project, err + return project, nil } func (c *Client) DeleteProject(project *Project) error { From 14fd7ce724a70535cea2088935a9d8b38ba58c34 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 23 Aug 2023 13:19:54 +0200 Subject: [PATCH 23/62] Update returns --- internal/serverless/client.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/serverless/client.go b/internal/serverless/client.go index 827e93e5c0..54e52c8a6a 100644 --- a/internal/serverless/client.go +++ b/internal/serverless/client.go @@ -193,7 +193,7 @@ func (c *Client) CreateProject(name, region, project string) (*Project, error) { return nil, fmt.Errorf("failed to reset credentials: %w", err) } - return serverlessProject, err + return serverlessProject, nil } func (c *Client) EnsureProjectInitialized(ctx context.Context, project *Project) error { @@ -220,7 +220,6 @@ func (c *Client) EnsureProjectInitialized(ctx context.Context, project *Project) return nil } - return nil } func (c *Client) StatusProject(ctx context.Context, project *Project) (string, error) { @@ -305,8 +304,11 @@ func (c *Client) GetProject(projectType, projectID string) (*Project, error) { project := &Project{url: c.host, apiKey: c.apiKey} err = json.Unmarshal(respBody, &project) + if err != nil { + return nil, fmt.Errorf("failed to decode project: %w", err) + } - return project, err + return project, nil } func (c *Client) EnsureEndpoints(ctx context.Context, project *Project) error { From 2060799dd0b8bf5046f8f5b7985128268640d3e1 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 23 Aug 2023 13:21:45 +0200 Subject: [PATCH 24/62] Keep just apiKey for serverless client struct --- internal/serverless/client.go | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/internal/serverless/client.go b/internal/serverless/client.go index 54e52c8a6a..8e9553ba74 100644 --- a/internal/serverless/client.go +++ b/internal/serverless/client.go @@ -23,9 +23,6 @@ import ( type Client struct { host string apiKey string - - username string - password string } // ClientOption is functional option modifying Serverless API client. @@ -74,20 +71,6 @@ func WithApiKey(apiKey string) ClientOption { } } -// WithUsername option sets the username. -func WithUsername(username string) ClientOption { - return func(c *Client) { - c.username = username - } -} - -// WithPassword option sets the password. -func WithPassword(password string) ClientOption { - return func(c *Client) { - c.password = password - } -} - func (c *Client) get(ctx context.Context, resourcePath string) (int, []byte, error) { return c.sendRequest(ctx, http.MethodGet, resourcePath, nil) } @@ -131,13 +114,8 @@ func (c *Client) newRequest(ctx context.Context, method, resourcePath string, re } req.Header.Add("content-type", "application/json") - - if c.username != "" { - req.SetBasicAuth(c.username, c.password) - return req, nil - } - req.Header.Add("Authorization", fmt.Sprintf("ApiKey %s", c.apiKey)) + return req, nil } From eea6fe356dc899f00712b1b81958f1ae158d2fda Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 23 Aug 2023 13:25:18 +0200 Subject: [PATCH 25/62] Add DefaultFleetServerURL comment --- internal/kibana/fleet.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/kibana/fleet.go b/internal/kibana/fleet.go index a55d41eda8..850fb7fc98 100644 --- a/internal/kibana/fleet.go +++ b/internal/kibana/fleet.go @@ -11,6 +11,7 @@ import ( "net/http" ) +// DefaultFleetServerURL returns the default Fleet server configured in Kibana func (c *Client) DefaultFleetServerURL() (string, error) { path := fmt.Sprintf("%s/fleet_server_hosts", FleetAPI) From cd3132a069e79ae9ef938b4b787de6751d8cec26 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 23 Aug 2023 19:32:37 +0200 Subject: [PATCH 26/62] Workaround to filter geo ip related keys --- .../testrunner/runners/pipeline/runner.go | 12 +++++++- .../runners/pipeline/test_result.go | 29 +++++++++++++++---- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/internal/testrunner/runners/pipeline/runner.go b/internal/testrunner/runners/pipeline/runner.go index 57d1d6f2bd..937040f52c 100644 --- a/internal/testrunner/runners/pipeline/runner.go +++ b/internal/testrunner/runners/pipeline/runner.go @@ -23,6 +23,7 @@ import ( "github.com/elastic/elastic-package/internal/multierror" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/signal" + "github.com/elastic/elastic-package/internal/stack" "github.com/elastic/elastic-package/internal/testrunner" ) @@ -291,7 +292,16 @@ func (r *runner) verifyResults(testCaseFile string, config *testConfig, result * } } - err := compareResults(testCasePath, config, result) + // TODO: currently GeoIP related fields are being removed + stackConfig, err := stack.LoadConfig(r.options.Profile) + if err != nil { + return err + } + skipGeoIP := false + if stackConfig.Provider == "serverless" { + skipGeoIP = true + } + err = compareResults(testCasePath, config, result, skipGeoIP) if _, ok := err.(testrunner.ErrTestCaseFailed); ok { return err } diff --git a/internal/testrunner/runners/pipeline/test_result.go b/internal/testrunner/runners/pipeline/test_result.go index 29c9fb8380..73af2f12e9 100644 --- a/internal/testrunner/runners/pipeline/test_result.go +++ b/internal/testrunner/runners/pipeline/test_result.go @@ -18,6 +18,7 @@ import ( "github.com/pmezard/go-difflib/difflib" "github.com/elastic/elastic-package/internal/common" + "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/testrunner" ) @@ -46,8 +47,8 @@ func writeTestResult(testCasePath string, result *testResult) error { return nil } -func compareResults(testCasePath string, config *testConfig, result *testResult) error { - resultsWithoutDynamicFields, err := adjustTestResult(result, config) +func compareResults(testCasePath string, config *testConfig, result *testResult, skipGeoip bool) error { + resultsWithoutDynamicFields, err := adjustTestResult(result, config, skipGeoip) if err != nil { return fmt.Errorf("can't adjust test results: %w", err) } @@ -57,7 +58,7 @@ func compareResults(testCasePath string, config *testConfig, result *testResult) return fmt.Errorf("marshalling actual test results failed: %w", err) } - expectedResults, err := readExpectedTestResult(testCasePath, config) + expectedResults, err := readExpectedTestResult(testCasePath, config, skipGeoip) if err != nil { return fmt.Errorf("reading expected test result failed: %w", err) } @@ -138,7 +139,7 @@ func diffJson(want, got []byte) (string, error) { return buf.String(), err } -func readExpectedTestResult(testCasePath string, config *testConfig) (*testResult, error) { +func readExpectedTestResult(testCasePath string, config *testConfig, skipGeoIP bool) (*testResult, error) { testCaseDir := filepath.Dir(testCasePath) testCaseFile := filepath.Base(testCasePath) @@ -153,14 +154,14 @@ func readExpectedTestResult(testCasePath string, config *testConfig) (*testResul return nil, fmt.Errorf("unmarshalling expected test result failed: %w", err) } - adjusted, err := adjustTestResult(u, config) + adjusted, err := adjustTestResult(u, config, skipGeoIP) if err != nil { return nil, fmt.Errorf("adjusting test result failed: %w", err) } return adjusted, nil } -func adjustTestResult(result *testResult, config *testConfig) (*testResult, error) { +func adjustTestResult(result *testResult, config *testConfig, skipGeoIP bool) (*testResult, error) { if config == nil || config.DynamicFields == nil { return result, nil } @@ -186,6 +187,22 @@ func adjustTestResult(result *testResult, config *testConfig) (*testResult, erro } } + if skipGeoIP { + // remove common related geoIP keys + geoIPKeys := []string{ + "source.as", + "source.geo", + "destination.as", + "destination.geo", + } + for _, key := range geoIPKeys { + err := m.Delete(key) + if err != nil && err != common.ErrKeyNotFound { + return nil, fmt.Errorf("can't remove geoIP field: %w", err) + } + } + } + b, err := json.Marshal(&m) if err != nil { return nil, fmt.Errorf("can't marshal event: %w", err) From ea1328fb1812ed0da460471a0eb9375fc4ea8e9b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 23 Aug 2023 19:43:39 +0200 Subject: [PATCH 27/62] Remove unused import --- internal/testrunner/runners/pipeline/test_result.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/testrunner/runners/pipeline/test_result.go b/internal/testrunner/runners/pipeline/test_result.go index 73af2f12e9..2d6fddea33 100644 --- a/internal/testrunner/runners/pipeline/test_result.go +++ b/internal/testrunner/runners/pipeline/test_result.go @@ -18,7 +18,6 @@ import ( "github.com/pmezard/go-difflib/difflib" "github.com/elastic/elastic-package/internal/common" - "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/testrunner" ) From 7a7793e0ecc887e0ac1f925cdc723d0e91d89ee9 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 25 Aug 2023 12:12:04 +0200 Subject: [PATCH 28/62] Changes from code review --- internal/serverless/client.go | 73 +++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/internal/serverless/client.go b/internal/serverless/client.go index 8e9553ba74..656854cffc 100644 --- a/internal/serverless/client.go +++ b/internal/serverless/client.go @@ -20,6 +20,8 @@ import ( "github.com/elastic/elastic-package/internal/logger" ) +const projectsAPI = "/api/v1/serverless/projects" + type Client struct { host string apiKey string @@ -29,19 +31,19 @@ type Client struct { type ClientOption func(*Client) var ( - ServerlessApiKeyEnvironmentVariable = "SERVERLESS_API_KEY" - ServerlessHostvironmentVariable = "SERVERLESS_HOST" + ServerlessApiKeyEnvironmentVariable = environment.WithElasticPackagePrefix("SERVERLESS_API_KEY") + ServerlessHostvironmentVariable = environment.WithElasticPackagePrefix("SERVERLESS_HOST") ErrProjectNotExist = errors.New("project does not exist") ) func NewClient(opts ...ClientOption) (*Client, error) { - hostEnvName := environment.WithElasticPackagePrefix(ServerlessHostvironmentVariable) + hostEnvName := ServerlessHostvironmentVariable host := os.Getenv(hostEnvName) if host == "" { return nil, fmt.Errorf("unable to obtain value from %s environment variable", hostEnvName) } - apiKeyEnvName := environment.WithElasticPackagePrefix(ServerlessApiKeyEnvironmentVariable) + apiKeyEnvName := ServerlessApiKeyEnvironmentVariable apiKey := os.Getenv(apiKeyEnvName) if apiKey == "" { return nil, fmt.Errorf("unable to obtain value from %s environment variable", apiKeyEnvName) @@ -136,7 +138,7 @@ func (c *Client) doRequest(request *http.Request) (int, []byte, error) { return resp.StatusCode, body, nil } -func (c *Client) CreateProject(name, region, project string) (*Project, error) { +func (c *Client) CreateProject(name, region, projectType string) (*Project, error) { ReqBody := struct { Name string `json:"name"` RegionID string `json:"region_id"` @@ -148,8 +150,11 @@ func (c *Client) CreateProject(name, region, project string) (*Project, error) { if err != nil { return nil, err } - ctx := context.Background() - resourcePath := fmt.Sprintf("%s/api/v1/serverless/projects/%s", c.host, project) + ctx := context.TODO() + resourcePath, err := url.JoinPath(c.host, projectsAPI, projectType) + if err != nil { + return nil, fmt.Errorf("could not build the URL: %w", err) + } statusCode, respBody, err := c.post(ctx, resourcePath, p) if err != nil { @@ -201,7 +206,10 @@ func (c *Client) EnsureProjectInitialized(ctx context.Context, project *Project) } func (c *Client) StatusProject(ctx context.Context, project *Project) (string, error) { - resourcePath := fmt.Sprintf("%s/api/v1/serverless/projects/%s/%s/status", c.host, project.Type, project.ID) + resourcePath, err := url.JoinPath(c.host, projectsAPI, project.Type, project.ID, "status") + if err != nil { + return "", fmt.Errorf("could not build the URL: %w", err) + } statusCode, respBody, err := c.get(ctx, resourcePath) if err != nil { @@ -224,7 +232,10 @@ func (c *Client) StatusProject(ctx context.Context, project *Project) (string, e } func (c *Client) ResetCredentials(ctx context.Context, project *Project) (*Project, error) { - resourcePath := fmt.Sprintf("%s/api/v1/serverless/projects/%s/%s/_reset-credentials", c.host, project.Type, project.ID) + resourcePath, err := url.JoinPath(c.host, projectsAPI, project.Type, project.ID, "_reset-credentials") + if err != nil { + return nil, fmt.Errorf("could not build the URL: %w", err) + } statusCode, respBody, err := c.post(ctx, resourcePath, nil) if err != nil { @@ -250,8 +261,11 @@ func (c *Client) ResetCredentials(ctx context.Context, project *Project) (*Proje } func (c *Client) DeleteProject(project *Project) error { - ctx := context.Background() - resourcePath := fmt.Sprintf("%s/api/v1/serverless/projects/%s/%s", c.host, project.Type, project.ID) + ctx := context.TODO() + resourcePath, err := url.JoinPath(c.host, projectsAPI, project.Type, project.ID) + if err != nil { + return fmt.Errorf("could not build the URL: %w", err) + } statusCode, _, err := c.delete(ctx, resourcePath) if err != nil { return fmt.Errorf("error deleting project: %w", err) @@ -265,8 +279,11 @@ func (c *Client) DeleteProject(project *Project) error { } func (c *Client) GetProject(projectType, projectID string) (*Project, error) { - ctx := context.Background() - resourcePath := fmt.Sprintf("%s/api/v1/serverless/projects/%s/%s", c.host, projectType, projectID) + ctx := context.TODO() + resourcePath, err := url.JoinPath(c.host, projectsAPI, projectType, projectID) + if err != nil { + return nil, fmt.Errorf("could not build the URL: %w", err) + } statusCode, respBody, err := c.get(ctx, resourcePath) if err != nil { return nil, fmt.Errorf("error deleting project: %w", err) @@ -290,26 +307,24 @@ func (c *Client) GetProject(projectType, projectID string) (*Project, error) { } func (c *Client) EnsureEndpoints(ctx context.Context, project *Project) error { - timer := time.NewTimer(time.Millisecond) + if project.Endpoints.Elasticsearch != "" { + return nil + } + for { + newProject, err := c.GetProject(project.Type, project.ID) + switch { + case err != nil: + logger.Debugf("request error: %s", err.Error()) + case newProject.Endpoints.Elasticsearch != "": + project.Endpoints = newProject.Endpoints + return nil + } + logger.Debugf("Waiting for Elasticsearch endpoint for %s project %q", project.Type, project.ID) select { case <-ctx.Done(): return ctx.Err() - case <-timer.C: + case <-time.After(time.Second * 5): } - - if project.Endpoints.Elasticsearch != "" { - return nil - } - - newProject, err := c.GetProject(project.Type, project.ID) - if err != nil { - logger.Debugf("request error: %s", err.Error()) - timer.Reset(time.Second * 5) - continue - } - - project.Endpoints = newProject.Endpoints - timer.Reset(time.Second * 5) } } From 899495ad66fc8c781c9d6893bc46533a9413639f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 25 Aug 2023 12:47:49 +0200 Subject: [PATCH 29/62] Changes from code review --- internal/serverless/project.go | 65 +++++++++---------- internal/stack/serverless.go | 1 - .../testrunner/runners/pipeline/runner.go | 4 +- 3 files changed, 31 insertions(+), 39 deletions(-) diff --git a/internal/serverless/project.go b/internal/serverless/project.go index 3dc9c35f9e..67b8f89880 100644 --- a/internal/serverless/project.go +++ b/internal/serverless/project.go @@ -48,7 +48,7 @@ type Project struct { } func (p *Project) EnsureHealthy(ctx context.Context) error { - if err := p.ensureElasticserchHealthy(ctx); err != nil { + if err := p.ensureElasticsearchHealthy(ctx); err != nil { return fmt.Errorf("elasticsearch not healthy: %w", err) } if err := p.ensureKibanaHealthy(ctx); err != nil { @@ -77,63 +77,51 @@ func (p *Project) Status(ctx context.Context) (map[string]string, error) { return status, nil } -func (p *Project) ensureElasticserchHealthy(ctx context.Context) error { - timer := time.NewTimer(time.Millisecond) +func (p *Project) ensureElasticsearchHealthy(ctx context.Context) error { for { + err := p.ElasticsearchClient.CheckHealth(ctx) + if err == nil { + return nil + } + + logger.Debugf("Elasticsearch service not ready: %s", err.Error()) select { case <-ctx.Done(): return ctx.Err() - case <-timer.C: - } - - err := p.ElasticsearchClient.CheckHealth(ctx) - if err != nil { - logger.Debugf("Elasticsearch service not ready: %s", err.Error()) - timer.Reset(time.Second * 5) - continue + case <-time.After(5 * time.Second): } - - return nil } } func (p *Project) ensureKibanaHealthy(ctx context.Context) error { - timer := time.NewTimer(time.Millisecond) for { + err := p.KibanaClient.CheckHealth() + if err == nil { + return nil + } + + logger.Debugf("Kibana service not ready: %s", err.Error()) select { case <-ctx.Done(): return ctx.Err() - case <-timer.C: + case <-time.After(5 * time.Second): } - - err := p.KibanaClient.CheckHealth() - if err != nil { - logger.Debugf("Kibana service not ready: %s", err.Error()) - timer.Reset(time.Second * 5) - continue - } - - return nil } } func (p *Project) ensureFleetHealthy(ctx context.Context) error { - timer := time.NewTimer(time.Millisecond) for { + err := p.getFleetHealth(ctx) + if err == nil { + return nil + } + + logger.Debugf("Fleet service not ready: %s", err.Error()) select { case <-ctx.Done(): return ctx.Err() - case <-timer.C: + case <-time.After(5 * time.Second): } - - err := p.getFleetHealth(ctx) - if err != nil { - logger.Debugf("Fleet service not ready: %s", err.Error()) - timer.Reset(time.Second * 5) - continue - } - - return nil } } @@ -231,7 +219,12 @@ func getPackageVersion(registryURL, packageName, stackVersion string) (string, e if err != nil { return "", fmt.Errorf("could not build URL: %w", err) } - searchURL = fmt.Sprintf("%s?package=%s&kibana.version=%s", searchURL, packageName, stackVersion) + // TODO: add capabilities or spec.minVersion? + queryValues := url.Values{} + queryValues.Add("package", packageName) + queryValues.Add("kibana.version", stackVersion) + + searchURL = fmt.Sprintf("%s?%s", searchURL, queryValues.Encode()) logger.Debugf("GET %s", searchURL) resp, err := http.Get(searchURL) if err != nil { diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 0ede2e5161..9923ed3d3a 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -54,7 +54,6 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op if err != nil { return Config{}, fmt.Errorf("failed to create %s project %s in %s: %w", settings.Type, settings.Name, settings.Region, err) } - // project, _ := sp.currentProject(conf) ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30) defer cancel() diff --git a/internal/testrunner/runners/pipeline/runner.go b/internal/testrunner/runners/pipeline/runner.go index 937040f52c..3c3e721779 100644 --- a/internal/testrunner/runners/pipeline/runner.go +++ b/internal/testrunner/runners/pipeline/runner.go @@ -292,13 +292,13 @@ func (r *runner) verifyResults(testCaseFile string, config *testConfig, result * } } - // TODO: currently GeoIP related fields are being removed + // TODO: currently GeoIP related fields are being removed when the serverless provider is used. stackConfig, err := stack.LoadConfig(r.options.Profile) if err != nil { return err } skipGeoIP := false - if stackConfig.Provider == "serverless" { + if stackConfig.Provider == stack.ProviderServerless { skipGeoIP = true } err = compareResults(testCasePath, config, result, skipGeoIP) From bf7d6f3e3e7c3cdfecb120c3c4a7f00044cffa9f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 25 Aug 2023 13:11:09 +0200 Subject: [PATCH 30/62] Add allowed serverless project types --- internal/stack/serverless.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 9923ed3d3a..06a4efe590 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -33,6 +33,11 @@ const ( ) var ( + allowedProjectTypes = map[string]struct{}{ + "security": {}, + "observability": {}, + } + errProjectNotExist = errors.New("project does not exist") ) @@ -216,7 +221,8 @@ func (sp *serverlessProvider) BootUp(options Options) error { return err } - if settings.Type == "elasticsearch" { + _, ok := allowedProjectTypes[settings.Type] + if !ok { return fmt.Errorf("serverless project type not supported: %s", settings.Type) } From e211658a48ca0e0207d0cf2b0f471585958eb629 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 25 Aug 2023 13:26:50 +0200 Subject: [PATCH 31/62] Remove leftovers --- internal/stack/serverless.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 06a4efe590..32512c686d 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -250,19 +250,12 @@ func (sp *serverlessProvider) BootUp(options Options) error { return fmt.Errorf("failed to create agent policy: %w", err) } + // TODO: Ensuring a specific GeoIP database would make tests reproducible + // Currently geo ip files would be ignored when running pipeline tests // logger.Infof("Replacing GeoIP databases") - // err = cp.replaceGeoIPDatabases(config, options, settings.TemplateID, settings.Region, payload.Resources.Elasticsearch[0].Plan.ClusterTopology) - // if err != nil { - // return fmt.Errorf("failed to replace GeoIP databases: %w", err) - // } case nil: logger.Debugf("%s project existed: %s", project.Type, project.Name) printUserConfig(options.Printer, config) - // logger.Infof("Updating project %s", project.Name) - // err = sp.updateDeployment(project, settings) - // if err != nil { - // return fmt.Errorf("failed to update deployment: %w", err) - // } } logger.Infof("Starting local agent") @@ -330,11 +323,8 @@ func (sp *serverlessProvider) TearDown(options Options) error { return fmt.Errorf("failed to delete project: %w", err) } + // TODO: if GeoIP database is specified, remove the geoip Bundle (if needed) // logger.Debugf("Deleting GeoIP bundle.") - // err = cp.deleteGeoIPExtension() - // if err != nil { - // return fmt.Errorf("failed to delete GeoIP extension: %w", err) - // } // err = storeConfig(sp.profile, Config{}) // if err != nil { From f68331e52e9393339a9b94d883e9cbde10cb4425 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 25 Aug 2023 15:59:19 +0200 Subject: [PATCH 32/62] Move clients to serverless provider --- internal/serverless/project.go | 41 +++++++++--------- internal/stack/serverless.go | 78 +++++++++++++++++++--------------- 2 files changed, 62 insertions(+), 57 deletions(-) diff --git a/internal/serverless/project.go b/internal/serverless/project.go index 67b8f89880..83fa8c3df6 100644 --- a/internal/serverless/project.go +++ b/internal/serverless/project.go @@ -42,16 +42,13 @@ type Project struct { Fleet string `json:"fleet,omitempty"` APM string `json:"apm,omitempty"` } `json:"endpoints"` - - ElasticsearchClient *elasticsearch.Client - KibanaClient *kibana.Client } -func (p *Project) EnsureHealthy(ctx context.Context) error { - if err := p.ensureElasticsearchHealthy(ctx); err != nil { +func (p *Project) EnsureHealthy(ctx context.Context, elasticsearchClient *elasticsearch.Client, kibanaClient *kibana.Client) error { + if err := p.ensureElasticsearchHealthy(ctx, elasticsearchClient); err != nil { return fmt.Errorf("elasticsearch not healthy: %w", err) } - if err := p.ensureKibanaHealthy(ctx); err != nil { + if err := p.ensureKibanaHealthy(ctx, kibanaClient); err != nil { return fmt.Errorf("kibana not healthy: %w", err) } if err := p.ensureFleetHealthy(ctx); err != nil { @@ -60,7 +57,7 @@ func (p *Project) EnsureHealthy(ctx context.Context) error { return nil } -func (p *Project) Status(ctx context.Context) (map[string]string, error) { +func (p *Project) Status(ctx context.Context, elasticsearchClient *elasticsearch.Client, kibanaClient *kibana.Client) (map[string]string, error) { var status map[string]string healthStatus := func(err error) string { if err != nil { @@ -70,16 +67,16 @@ func (p *Project) Status(ctx context.Context) (map[string]string, error) { } status = map[string]string{ - "elasticsearch": healthStatus(p.getESHealth(ctx)), - "kibana": healthStatus(p.getKibanaHealth()), + "elasticsearch": healthStatus(p.getESHealth(ctx, elasticsearchClient)), + "kibana": healthStatus(p.getKibanaHealth(kibanaClient)), "fleet": healthStatus(p.getFleetHealth(ctx)), } return status, nil } -func (p *Project) ensureElasticsearchHealthy(ctx context.Context) error { +func (p *Project) ensureElasticsearchHealthy(ctx context.Context, elasticsearchClient *elasticsearch.Client) error { for { - err := p.ElasticsearchClient.CheckHealth(ctx) + err := elasticsearchClient.CheckHealth(ctx) if err == nil { return nil } @@ -93,9 +90,9 @@ func (p *Project) ensureElasticsearchHealthy(ctx context.Context) error { } } -func (p *Project) ensureKibanaHealthy(ctx context.Context) error { +func (p *Project) ensureKibanaHealthy(ctx context.Context, kibanaClient *kibana.Client) error { for { - err := p.KibanaClient.CheckHealth() + err := kibanaClient.CheckHealth() if err == nil { return nil } @@ -125,8 +122,8 @@ func (p *Project) ensureFleetHealthy(ctx context.Context) error { } } -func (p *Project) DefaultFleetServerURL() (string, error) { - fleetURL, err := p.KibanaClient.DefaultFleetServerURL() +func (p *Project) DefaultFleetServerURL(kibanaClient *kibana.Client) (string, error) { + fleetURL, err := kibanaClient.DefaultFleetServerURL() if err != nil { return "", fmt.Errorf("failed to query fleet server hosts: %w", err) } @@ -134,12 +131,12 @@ func (p *Project) DefaultFleetServerURL() (string, error) { return fleetURL, nil } -func (p *Project) getESHealth(ctx context.Context) error { - return p.ElasticsearchClient.CheckHealth(ctx) +func (p *Project) getESHealth(ctx context.Context, elasticsearchClient *elasticsearch.Client) error { + return elasticsearchClient.CheckHealth(ctx) } -func (p *Project) getKibanaHealth() error { - return p.KibanaClient.CheckHealth() +func (p *Project) getKibanaHealth(kibanaClient *kibana.Client) error { + return kibanaClient.CheckHealth() } func (p *Project) getFleetHealth(ctx context.Context) error { @@ -180,7 +177,7 @@ func (p *Project) getFleetHealth(ctx context.Context) error { return nil } -func (p *Project) CreateAgentPolicy(stackVersion string) error { +func (p *Project) CreateAgentPolicy(stackVersion string, kibanaClient *kibana.Client) error { systemVersion, err := getPackageVersion(eprURL, "system", stackVersion) if err != nil { return fmt.Errorf("could not get the system package version for kibana %v: %w", stackVersion, err) @@ -193,7 +190,7 @@ func (p *Project) CreateAgentPolicy(stackVersion string) error { Namespace: "default", MonitoringEnabled: []string{"logs", "metrics"}, } - newPolicy, err := p.KibanaClient.CreatePolicy(policy) + newPolicy, err := kibanaClient.CreatePolicy(policy) if err != nil { return fmt.Errorf("error while creating agent policy: %w", err) } @@ -206,7 +203,7 @@ func (p *Project) CreateAgentPolicy(stackVersion string) error { packagePolicy.Package.Name = "system" packagePolicy.Package.Version = systemVersion - _, err = p.KibanaClient.CreatePackagePolicy(packagePolicy) + _, err = kibanaClient.CreatePackagePolicy(packagePolicy) if err != nil { return fmt.Errorf("error while creating package policy: %w", err) } diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 32512c686d..d69c1df64d 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -44,6 +44,9 @@ var ( type serverlessProvider struct { profile *profile.Profile client *serverless.Client + + elasticsearchClient *elasticsearch.Client + kibanaClient *kibana.Client } type projectSettings struct { @@ -60,7 +63,7 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op return Config{}, fmt.Errorf("failed to create %s project %s in %s: %w", settings.Type, settings.Name, settings.Region, err) } - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30) + ctx, cancel := context.WithTimeout(context.TODO(), time.Minute*30) defer cancel() if err := sp.client.EnsureEndpoints(ctx, project); err != nil { return Config{}, fmt.Errorf("failed to ensure endpoints have been provisioned properly: %w", err) @@ -91,12 +94,12 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op return Config{}, fmt.Errorf("project not initialized: %w", err) } - project, err = sp.createClients(project) + err = sp.createClients(project) if err != nil { - return Config{}, fmt.Errorf("failed to create project client") + return Config{}, err } - config.Parameters[paramServerlessFleetURL], err = project.DefaultFleetServerURL() + config.Parameters[paramServerlessFleetURL], err = project.DefaultFleetServerURL(sp.kibanaClient) if err != nil { return Config{}, fmt.Errorf("failed to get fleet URL: %w", err) } @@ -110,7 +113,7 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op return Config{}, fmt.Errorf("failed to store config: %w", err) } - err = project.EnsureHealthy(ctx) + err = project.EnsureHealthy(ctx, sp.elasticsearchClient, sp.kibanaClient) if err != nil { return Config{}, fmt.Errorf("not all services are healthy: %w", err) } @@ -118,29 +121,6 @@ func (sp *serverlessProvider) createProject(settings projectSettings, options Op return config, nil } -func (sp *serverlessProvider) createClients(project *serverless.Project) (*serverless.Project, error) { - var err error - project.ElasticsearchClient, err = NewElasticsearchClient( - elasticsearch.OptionWithAddress(project.Endpoints.Elasticsearch), - elasticsearch.OptionWithUsername(project.Credentials.Username), - elasticsearch.OptionWithPassword(project.Credentials.Password), - ) - if err != nil { - return project, fmt.Errorf("failed to create elasticsearch client") - } - - project.KibanaClient, err = NewKibanaClient( - kibana.Address(project.Endpoints.Kibana), - kibana.Username(project.Credentials.Username), - kibana.Password(project.Credentials.Password), - ) - if err != nil { - return project, fmt.Errorf("failed to create kibana client") - } - - return project, nil -} - func (sp *serverlessProvider) deleteProject(project *serverless.Project, options Options) error { return sp.client.DeleteProject(project) } @@ -167,14 +147,14 @@ func (sp *serverlessProvider) currentProject(config Config) (*serverless.Project project.Credentials.Username = config.ElasticsearchUsername project.Credentials.Password = config.ElasticsearchPassword - project, err = sp.createClients(project) + err = sp.createClients(project) if err != nil { - return nil, fmt.Errorf("failed to create project client") + return nil, err } fleetURL := config.Parameters[paramServerlessFleetURL] if true { - fleetURL, err = project.DefaultFleetServerURL() + fleetURL, err = project.DefaultFleetServerURL(sp.kibanaClient) if err != nil { return nil, fmt.Errorf("failed to get fleet URL: %w", err) } @@ -184,6 +164,29 @@ func (sp *serverlessProvider) currentProject(config Config) (*serverless.Project return project, nil } +func (sp *serverlessProvider) createClients(project *serverless.Project) error { + var err error + sp.elasticsearchClient, err = NewElasticsearchClient( + elasticsearch.OptionWithAddress(project.Endpoints.Elasticsearch), + elasticsearch.OptionWithUsername(project.Credentials.Username), + elasticsearch.OptionWithPassword(project.Credentials.Password), + ) + if err != nil { + return fmt.Errorf("failed to create elasticsearch client") + } + + sp.kibanaClient, err = NewKibanaClient( + kibana.Address(project.Endpoints.Kibana), + kibana.Username(project.Credentials.Username), + kibana.Password(project.Credentials.Password), + ) + if err != nil { + return fmt.Errorf("failed to create kibana client") + } + + return nil +} + func getProjectSettings(options Options) (projectSettings, error) { s := projectSettings{ Name: createProjectName(options), @@ -205,7 +208,7 @@ func newServerlessProvider(profile *profile.Profile) (*serverlessProvider, error return nil, fmt.Errorf("can't create serverless provider: %w", err) } - return &serverlessProvider{profile, client}, nil + return &serverlessProvider{profile, client, nil, nil}, nil } func (sp *serverlessProvider) BootUp(options Options) error { @@ -244,8 +247,13 @@ func (sp *serverlessProvider) BootUp(options Options) error { return fmt.Errorf("failed to retrieve latest project created: %w", err) } + err = sp.createClients(project) + if err != nil { + return err + } + logger.Infof("Creating agent policy") - err = project.CreateAgentPolicy(options.StackVersion) + err = project.CreateAgentPolicy(options.StackVersion, sp.kibanaClient) if err != nil { return fmt.Errorf("failed to create agent policy: %w", err) } @@ -368,8 +376,8 @@ func (sp *serverlessProvider) Status(options Options) ([]ServiceStatus, error) { return nil, err } - ctx := context.Background() - projectServiceStatus, err := project.Status(ctx) + ctx := context.TODO() + projectServiceStatus, err := project.Status(ctx, sp.elasticsearchClient, sp.kibanaClient) if err != nil { return nil, err } From 472095f1fdfd5a251338f5fce39729ec0d9255a4 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 25 Aug 2023 16:58:01 +0200 Subject: [PATCH 33/62] Reuse elastic agent env template --- internal/stack/_static/elastic-agent.env.tmpl | 8 +++++--- .../stack/_static/serverless-elastic-agent.env.tmpl | 10 ---------- .../stack/_static/serverless-elastic-agent.yml.tmpl | 2 +- internal/stack/resources.go | 3 +++ internal/stack/serverlessresources.go | 8 ++------ 5 files changed, 11 insertions(+), 20 deletions(-) delete mode 100644 internal/stack/_static/serverless-elastic-agent.env.tmpl diff --git a/internal/stack/_static/elastic-agent.env.tmpl b/internal/stack/_static/elastic-agent.env.tmpl index 70a0b5cb58..d8888122e1 100644 --- a/internal/stack/_static/elastic-agent.env.tmpl +++ b/internal/stack/_static/elastic-agent.env.tmpl @@ -1,8 +1,10 @@ {{ $version := fact "agent_version" }} FLEET_ENROLL=1 -FLEET_URL=https://fleet-server:8220 -KIBANA_FLEET_HOST=https://kibana:5601 -KIBANA_HOST=https://kibana:5601 +FLEET_URL={{ fact "fleet_url" }} +KIBANA_FLEET_HOST={{ fact "kibana_host" }} +KIBANA_HOST={{ fact "kibana_host" }} +ELASTICSEARCH_USERNAME={{ fact "username" }} +ELASTICSEARCH_PASSWORD={{ fact "password" }} {{ if not (semverLessThan $version "8.0.0") }} FLEET_TOKEN_POLICY_NAME=Elastic-Agent (elastic-package) {{ end }} diff --git a/internal/stack/_static/serverless-elastic-agent.env.tmpl b/internal/stack/_static/serverless-elastic-agent.env.tmpl deleted file mode 100644 index d8888122e1..0000000000 --- a/internal/stack/_static/serverless-elastic-agent.env.tmpl +++ /dev/null @@ -1,10 +0,0 @@ -{{ $version := fact "agent_version" }} -FLEET_ENROLL=1 -FLEET_URL={{ fact "fleet_url" }} -KIBANA_FLEET_HOST={{ fact "kibana_host" }} -KIBANA_HOST={{ fact "kibana_host" }} -ELASTICSEARCH_USERNAME={{ fact "username" }} -ELASTICSEARCH_PASSWORD={{ fact "password" }} -{{ if not (semverLessThan $version "8.0.0") }} -FLEET_TOKEN_POLICY_NAME=Elastic-Agent (elastic-package) -{{ end }} diff --git a/internal/stack/_static/serverless-elastic-agent.yml.tmpl b/internal/stack/_static/serverless-elastic-agent.yml.tmpl index 150cff7ccb..94744bb2b7 100644 --- a/internal/stack/_static/serverless-elastic-agent.yml.tmpl +++ b/internal/stack/_static/serverless-elastic-agent.yml.tmpl @@ -9,7 +9,7 @@ services: retries: 180 interval: 5s hostname: docker-fleet-agent - env_file: "./serverless-elastic-agent.env" + env_file: "./elastic-agent.env" volumes: - type: bind source: ../../../tmp/service_logs/ diff --git a/internal/stack/resources.go b/internal/stack/resources.go index 9a235ac142..05701c0587 100644 --- a/internal/stack/resources.go +++ b/internal/stack/resources.go @@ -116,6 +116,9 @@ func applyResources(profile *profile.Profile, stackVersion string) error { "kibana_version": stackVersion, "agent_version": stackVersion, + "kibana_host": "https://kibana:5601", + "fleet_url": "https://fleet-server:8220", + "username": elasticsearchUsername, "password": elasticsearchPassword, diff --git a/internal/stack/serverlessresources.go b/internal/stack/serverlessresources.go index e29e616c0d..3e01837d51 100644 --- a/internal/stack/serverlessresources.go +++ b/internal/stack/serverlessresources.go @@ -17,10 +17,6 @@ import ( ) const ( - // ServerlessElasticAgentEnvFile is the elastic agent environment variables file for the - // serverless provider. - ServerlessElasticAgentEnvFile = "serverless-elastic-agent.env" - // ServerlessComposeFile is the docker-compose snapshot.yml file name. ServerlessComposeFile = "serverless-elastic-agent.yml" ) @@ -32,8 +28,8 @@ var ( Content: staticSource.Template("_static/serverless-elastic-agent.yml.tmpl"), }, &resource.File{ - Path: ServerlessElasticAgentEnvFile, - Content: staticSource.Template("_static/serverless-elastic-agent.env.tmpl"), + Path: ElasticAgentEnvFile, + Content: staticSource.Template("_static/elastic-agent.env.tmpl"), }, } ) From d09f0d775f8848dfaf5edf82f7b267c4dc6d8176 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 25 Aug 2023 17:33:00 +0200 Subject: [PATCH 34/62] Update stack templates with fleet_url and kibana_host --- internal/stack/_static/docker-compose-stack.yml.tmpl | 6 +++--- internal/stack/_static/kibana.yml.tmpl | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/stack/_static/docker-compose-stack.yml.tmpl b/internal/stack/_static/docker-compose-stack.yml.tmpl index 141448eee8..d1033c19cc 100644 --- a/internal/stack/_static/docker-compose-stack.yml.tmpl +++ b/internal/stack/_static/docker-compose-stack.yml.tmpl @@ -102,11 +102,11 @@ services: - "FLEET_SERVER_ENABLE=1" - "FLEET_SERVER_HOST=0.0.0.0" - "FLEET_SERVER_SERVICE_TOKEN=AAEAAWVsYXN0aWMvZmxlZXQtc2VydmVyL2VsYXN0aWMtcGFja2FnZS1mbGVldC1zZXJ2ZXItdG9rZW46bmgtcFhoQzRRQ2FXbms2U0JySGlWQQ" - - "FLEET_URL=https://fleet-server:8220" - - "KIBANA_FLEET_HOST=https://kibana:5601" + - "FLEET_URL={{ fact "fleet_url" }}" + - "KIBANA_FLEET_HOST={{ fact "kibana_host" }}" - "KIBANA_FLEET_SERVICE_TOKEN=AAEAAWVsYXN0aWMvZmxlZXQtc2VydmVyL2VsYXN0aWMtcGFja2FnZS1mbGVldC1zZXJ2ZXItdG9rZW46bmgtcFhoQzRRQ2FXbms2U0JySGlWQQ" - "KIBANA_FLEET_SETUP=1" - - "KIBANA_HOST=https://kibana:5601" + - "KIBANA_HOST={{ fact "kibana_host" }}" volumes: - "../certs/ca-cert.pem:/etc/ssl/certs/elastic-package.pem" - "../certs/fleet-server:/etc/ssl/elastic-agent" diff --git a/internal/stack/_static/kibana.yml.tmpl b/internal/stack/_static/kibana.yml.tmpl index 9e52be0d8a..80b908c0f9 100644 --- a/internal/stack/_static/kibana.yml.tmpl +++ b/internal/stack/_static/kibana.yml.tmpl @@ -26,7 +26,7 @@ xpack.fleet.agents.elasticsearch.hosts: ["https://elasticsearch:9200"] xpack.fleet.registryUrl: "https://package-registry:8080" xpack.fleet.agents.enabled: true -xpack.fleet.agents.fleet_server.hosts: ["https://fleet-server:8220"] +xpack.fleet.agents.fleet_server.hosts: ["{{ fact "fleet_url" }}"] {{ if and (not (semverLessThan $version "8.7.0")) (semverLessThan $version "8.10.0-SNAPSHOT") }} xpack.fleet.enableExperimental: ["experimentalDataStreamSettings"] # Enable experimental toggles in Fleet UI From 995a14a470e9dd4c7627c2dcc5d1af989be60b09 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 25 Aug 2023 18:02:38 +0200 Subject: [PATCH 35/62] Updated geo ip keys from ECS --- .../runners/pipeline/test_result.go | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/internal/testrunner/runners/pipeline/test_result.go b/internal/testrunner/runners/pipeline/test_result.go index 2d6fddea33..2a53eb5910 100644 --- a/internal/testrunner/runners/pipeline/test_result.go +++ b/internal/testrunner/runners/pipeline/test_result.go @@ -23,6 +23,23 @@ import ( const expectedTestResultSuffix = "-expected.json" +var geoIPKeys = []string{ + "client.as", + "client.geo", + "destination.as", + "destination.geo", + "host.geo", // not defined host.as in ECS + "observer.geo", // not defined observer.as in ECS + "server.as", + "server.geo", + "source.as", + "source.geo", + "threat.enrichments.indicateor.as", + "threat.enrichments.indicateor.geo", + "threat.indicateor.as", + "threat.indicateor.geo", +} + type testResult struct { events []json.RawMessage } @@ -187,13 +204,6 @@ func adjustTestResult(result *testResult, config *testConfig, skipGeoIP bool) (* } if skipGeoIP { - // remove common related geoIP keys - geoIPKeys := []string{ - "source.as", - "source.geo", - "destination.as", - "destination.geo", - } for _, key := range geoIPKeys { err := m.Delete(key) if err != nil && err != common.ErrKeyNotFound { From 5c62733e674d9302d986987b5896f6b80758ab75 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 25 Aug 2023 18:15:29 +0200 Subject: [PATCH 36/62] No need to return project in ResetCredentials --- internal/serverless/client.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/serverless/client.go b/internal/serverless/client.go index 656854cffc..2acee2b2dc 100644 --- a/internal/serverless/client.go +++ b/internal/serverless/client.go @@ -171,7 +171,7 @@ func (c *Client) CreateProject(name, region, projectType string) (*Project, erro return nil, fmt.Errorf("error while decoding create project response: %w", err) } - serverlessProject, err = c.ResetCredentials(ctx, serverlessProject) + err = c.ResetCredentials(ctx, serverlessProject) if err != nil { return nil, fmt.Errorf("failed to reset credentials: %w", err) } @@ -231,19 +231,19 @@ func (c *Client) StatusProject(ctx context.Context, project *Project) (string, e return status.Phase, nil } -func (c *Client) ResetCredentials(ctx context.Context, project *Project) (*Project, error) { +func (c *Client) ResetCredentials(ctx context.Context, project *Project) error { resourcePath, err := url.JoinPath(c.host, projectsAPI, project.Type, project.ID, "_reset-credentials") if err != nil { - return nil, fmt.Errorf("could not build the URL: %w", err) + return fmt.Errorf("could not build the URL: %w", err) } statusCode, respBody, err := c.post(ctx, resourcePath, nil) if err != nil { - return nil, fmt.Errorf("error creating project: %w", err) + return fmt.Errorf("error creating project: %w", err) } if statusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code %d", statusCode) + return fmt.Errorf("unexpected status code %d", statusCode) } var credentials struct { @@ -251,13 +251,13 @@ func (c *Client) ResetCredentials(ctx context.Context, project *Project) (*Proje Password string `json:"password"` } if err := json.Unmarshal(respBody, &credentials); err != nil { - return nil, fmt.Errorf("unable to decode credentials: %w", err) + return fmt.Errorf("unable to decode credentials: %w", err) } project.Credentials.Username = credentials.Username project.Credentials.Password = credentials.Password - return project, nil + return nil } func (c *Client) DeleteProject(project *Project) error { From 1165923a77478659144647995ad10da747ee9739 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 31 Aug 2023 13:10:39 +0200 Subject: [PATCH 37/62] Use registry.Production client --- internal/serverless/project.go | 53 ++++++---------------------------- internal/stack/serverless.go | 4 +-- 2 files changed, 11 insertions(+), 46 deletions(-) diff --git a/internal/serverless/project.go b/internal/serverless/project.go index 83fa8c3df6..f44c07505b 100644 --- a/internal/serverless/project.go +++ b/internal/serverless/project.go @@ -16,6 +16,7 @@ import ( "github.com/elastic/elastic-package/internal/elasticsearch" "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/registry" ) const eprURL = "https://epr.elastic.co" @@ -178,10 +179,16 @@ func (p *Project) getFleetHealth(ctx context.Context) error { } func (p *Project) CreateAgentPolicy(stackVersion string, kibanaClient *kibana.Client) error { - systemVersion, err := getPackageVersion(eprURL, "system", stackVersion) + systemPackages, err := registry.Production.Revisions("system", registry.SearchOptions{ + KibanaVersion: stackVersion, + }) if err != nil { return fmt.Errorf("could not get the system package version for kibana %v: %w", stackVersion, err) } + if len(systemPackages) != 1 { + return fmt.Errorf("unexpected number of system package versions - found %d expected 1", len(systemPackages)) + } + logger.Debugf("Found %s package - version %s", systemPackages[0].Name, systemPackages[0].Version) policy := kibana.Policy{ ID: "elastic-agent-managed-ep", @@ -201,7 +208,7 @@ func (p *Project) CreateAgentPolicy(stackVersion string, kibanaClient *kibana.Cl Namespace: newPolicy.Namespace, } packagePolicy.Package.Name = "system" - packagePolicy.Package.Version = systemVersion + packagePolicy.Package.Version = systemPackages[0].Version _, err = kibanaClient.CreatePackagePolicy(packagePolicy) if err != nil { @@ -210,45 +217,3 @@ func (p *Project) CreateAgentPolicy(stackVersion string, kibanaClient *kibana.Cl return nil } - -func getPackageVersion(registryURL, packageName, stackVersion string) (string, error) { - searchURL, err := url.JoinPath(registryURL, "search") - if err != nil { - return "", fmt.Errorf("could not build URL: %w", err) - } - // TODO: add capabilities or spec.minVersion? - queryValues := url.Values{} - queryValues.Add("package", packageName) - queryValues.Add("kibana.version", stackVersion) - - searchURL = fmt.Sprintf("%s?%s", searchURL, queryValues.Encode()) - logger.Debugf("GET %s", searchURL) - resp, err := http.Get(searchURL) - if err != nil { - return "", fmt.Errorf("request failed (url: %s): %w", searchURL, err) - } - defer resp.Body.Close() - if resp.StatusCode >= 300 { - return "", fmt.Errorf("unexpected status code %v", resp.StatusCode) - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("failed to read response body: %w", err) - } - var packages []struct { - Name string `json:"name"` - Version string `json:"version"` - } - err = json.Unmarshal(body, &packages) - if err != nil { - return "", fmt.Errorf("failed to parse response body: %w", err) - } - if len(packages) != 1 { - return "", fmt.Errorf("expected 1 package, obtained %v", len(packages)) - } - if found := packages[0].Name; found != packageName { - return "", fmt.Errorf("expected package %s, found %s", packageName, found) - } - - return packages[0].Version, nil -} diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index d69c1df64d..30be31166e 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -172,7 +172,7 @@ func (sp *serverlessProvider) createClients(project *serverless.Project) error { elasticsearch.OptionWithPassword(project.Credentials.Password), ) if err != nil { - return fmt.Errorf("failed to create elasticsearch client") + return fmt.Errorf("failed to create elasticsearch client: %w", err) } sp.kibanaClient, err = NewKibanaClient( @@ -181,7 +181,7 @@ func (sp *serverlessProvider) createClients(project *serverless.Project) error { kibana.Password(project.Credentials.Password), ) if err != nil { - return fmt.Errorf("failed to create kibana client") + return fmt.Errorf("failed to create kibana client: %w", err) } return nil From 97b5a4f1ea6c86c3861c89fa81778463c999c577 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 31 Aug 2023 17:12:12 +0200 Subject: [PATCH 38/62] Remove unused var --- internal/serverless/project.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/serverless/project.go b/internal/serverless/project.go index f44c07505b..32a4487551 100644 --- a/internal/serverless/project.go +++ b/internal/serverless/project.go @@ -19,8 +19,6 @@ import ( "github.com/elastic/elastic-package/internal/registry" ) -const eprURL = "https://epr.elastic.co" - // Project represents a serverless project type Project struct { url string From 2644090743a2e26fe86dea704cc3c7f4a85eb622 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 4 Sep 2023 16:34:05 +0200 Subject: [PATCH 39/62] Update region --- internal/stack/serverless.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 30be31166e..aac757d26d 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -28,7 +28,7 @@ const ( configRegion = "stack.serverless.region" configProjectType = "stack.serverless.type" - defaultRegion = "aws-eu-west-1" + defaultRegion = "aws-us-east-1" defaultProjectType = "observability" ) From aa3f8d4dd93cab7b771a34937af7a5b421f0003d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 4 Sep 2023 16:35:16 +0200 Subject: [PATCH 40/62] Add config examples in profile --- internal/profile/_static/config.yml.example | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/profile/_static/config.yml.example b/internal/profile/_static/config.yml.example index 9146275871..14b3cf0687 100644 --- a/internal/profile/_static/config.yml.example +++ b/internal/profile/_static/config.yml.example @@ -1,2 +1,4 @@ # Directory containing GeoIP databases for stacks managed by elastic-agent. # stack.geoip_dir: "/path/to/geoip_dir/" +# stack.serverless.type: observability +# stack.serverless.region: aws-us-east-1 From efa4f7608967e08615ae55d3b473bb2a43a5c9fd Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 4 Sep 2023 17:44:47 +0200 Subject: [PATCH 41/62] Allow to loop for every event even with empty config --- .../testrunner/runners/pipeline/test_result.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/internal/testrunner/runners/pipeline/test_result.go b/internal/testrunner/runners/pipeline/test_result.go index 2a53eb5910..edce441421 100644 --- a/internal/testrunner/runners/pipeline/test_result.go +++ b/internal/testrunner/runners/pipeline/test_result.go @@ -178,11 +178,6 @@ func readExpectedTestResult(testCasePath string, config *testConfig, skipGeoIP b } func adjustTestResult(result *testResult, config *testConfig, skipGeoIP bool) (*testResult, error) { - if config == nil || config.DynamicFields == nil { - return result, nil - } - - // Strip dynamic fields from test result var stripped testResult for _, event := range result.events { if event == nil { @@ -196,10 +191,13 @@ func adjustTestResult(result *testResult, config *testConfig, skipGeoIP bool) (* return nil, fmt.Errorf("can't unmarshal event: %s: %w", string(event), err) } - for key := range config.DynamicFields { - err := m.Delete(key) - if err != nil && err != common.ErrKeyNotFound { - return nil, fmt.Errorf("can't remove dynamic field: %w", err) + if config != nil && config.DynamicFields != nil { + // Strip dynamic fields from test result + for key := range config.DynamicFields { + err := m.Delete(key) + if err != nil && err != common.ErrKeyNotFound { + return nil, fmt.Errorf("can't remove dynamic field: %w", err) + } } } From cc85e3cc816bed563ba4a8952637e587e320f3a4 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 4 Sep 2023 17:45:09 +0200 Subject: [PATCH 42/62] Add two more GeoIP keys to be skipped --- internal/testrunner/runners/pipeline/test_result.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/testrunner/runners/pipeline/test_result.go b/internal/testrunner/runners/pipeline/test_result.go index edce441421..9ea1a98706 100644 --- a/internal/testrunner/runners/pipeline/test_result.go +++ b/internal/testrunner/runners/pipeline/test_result.go @@ -24,6 +24,8 @@ import ( const expectedTestResultSuffix = "-expected.json" var geoIPKeys = []string{ + "as", + "geo", "client.as", "client.geo", "destination.as", From 948f902bd495dd2c05bf710c0642ec6dc9d9b6b5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 4 Sep 2023 17:55:50 +0200 Subject: [PATCH 43/62] Return earlier if there is no need to loop for every event --- internal/testrunner/runners/pipeline/test_result.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/testrunner/runners/pipeline/test_result.go b/internal/testrunner/runners/pipeline/test_result.go index 9ea1a98706..f544c9ebd7 100644 --- a/internal/testrunner/runners/pipeline/test_result.go +++ b/internal/testrunner/runners/pipeline/test_result.go @@ -180,6 +180,9 @@ func readExpectedTestResult(testCasePath string, config *testConfig, skipGeoIP b } func adjustTestResult(result *testResult, config *testConfig, skipGeoIP bool) (*testResult, error) { + if !skipGeoIP && (config == nil || config.DynamicFields == nil) { + return result, nil + } var stripped testResult for _, event := range result.events { if event == nil { From 9b669bf0151bf426c6e8e11318a3940432eeb57f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 5 Sep 2023 10:56:45 +0200 Subject: [PATCH 44/62] Not return error if project does not exist - status command --- internal/stack/serverless.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index aac757d26d..e503e8fd73 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -260,7 +260,6 @@ func (sp *serverlessProvider) BootUp(options Options) error { // TODO: Ensuring a specific GeoIP database would make tests reproducible // Currently geo ip files would be ignored when running pipeline tests - // logger.Infof("Replacing GeoIP databases") case nil: logger.Debugf("%s project existed: %s", project.Type, project.Name) printUserConfig(options.Printer, config) @@ -332,13 +331,6 @@ func (sp *serverlessProvider) TearDown(options Options) error { } // TODO: if GeoIP database is specified, remove the geoip Bundle (if needed) - // logger.Debugf("Deleting GeoIP bundle.") - - // err = storeConfig(sp.profile, Config{}) - // if err != nil { - // return fmt.Errorf("failed to store config: %w", err) - // } - return nil } @@ -372,6 +364,9 @@ func (sp *serverlessProvider) Status(options Options) ([]ServiceStatus, error) { } project, err := sp.currentProject(config) + if errors.Is(errProjectNotExist, err) { + return nil, nil + } if err != nil { return nil, err } From 44e3b48df21c406fec21f530f1f710dffd7fa070 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 5 Sep 2023 11:09:48 +0200 Subject: [PATCH 45/62] Rename environment variables --- internal/serverless/client.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/serverless/client.go b/internal/serverless/client.go index 2acee2b2dc..7e372d9145 100644 --- a/internal/serverless/client.go +++ b/internal/serverless/client.go @@ -16,7 +16,6 @@ import ( "os" "time" - "github.com/elastic/elastic-package/internal/environment" "github.com/elastic/elastic-package/internal/logger" ) @@ -31,8 +30,8 @@ type Client struct { type ClientOption func(*Client) var ( - ServerlessApiKeyEnvironmentVariable = environment.WithElasticPackagePrefix("SERVERLESS_API_KEY") - ServerlessHostvironmentVariable = environment.WithElasticPackagePrefix("SERVERLESS_HOST") + ServerlessApiKeyEnvironmentVariable = "EC_API_KEY" + ServerlessHostvironmentVariable = "EC_HOST" ErrProjectNotExist = errors.New("project does not exist") ) From ed9c77aa114ca3da9f16b125f100380a787e5b91 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 5 Sep 2023 11:37:16 +0200 Subject: [PATCH 46/62] Add default host URL --- internal/serverless/client.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/serverless/client.go b/internal/serverless/client.go index 7e372d9145..bece791a28 100644 --- a/internal/serverless/client.go +++ b/internal/serverless/client.go @@ -19,7 +19,11 @@ import ( "github.com/elastic/elastic-package/internal/logger" ) -const projectsAPI = "/api/v1/serverless/projects" +const ( + defaultHostURL = "https://cloud.elastic.co" + + projectsAPI = "/api/v1/serverless/projects" +) type Client struct { host string @@ -38,7 +42,10 @@ var ( func NewClient(opts ...ClientOption) (*Client, error) { hostEnvName := ServerlessHostvironmentVariable - host := os.Getenv(hostEnvName) + host, ok := os.LookupEnv(hostEnvName) + if !ok { + host = defaultHostURL + } if host == "" { return nil, fmt.Errorf("unable to obtain value from %s environment variable", hostEnvName) } From e6954772fc56c558942e110f564477b83448f12d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 5 Sep 2023 11:39:02 +0200 Subject: [PATCH 47/62] Use serverless.ErrPrtojectNotExist error --- internal/stack/serverless.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index e503e8fd73..445734fd38 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -37,8 +37,6 @@ var ( "security": {}, "observability": {}, } - - errProjectNotExist = errors.New("project does not exist") ) type serverlessProvider struct { @@ -128,17 +126,17 @@ func (sp *serverlessProvider) deleteProject(project *serverless.Project, options func (sp *serverlessProvider) currentProject(config Config) (*serverless.Project, error) { projectID, found := config.Parameters[paramServerlessProjectID] if !found { - return nil, errProjectNotExist + return nil, fmt.Errorf("mssing serverless project id") } projectType, found := config.Parameters[paramServerlessProjectType] if !found { - return nil, errProjectNotExist + return nil, fmt.Errorf("missing serverless project type") } project, err := sp.client.GetProject(projectType, projectID) - if err == serverless.ErrProjectNotExist { - return nil, errProjectNotExist + if errors.Is(serverless.ErrProjectNotExist, err) { + return nil, err } if err != nil { return nil, fmt.Errorf("couldn't check project health: %w", err) @@ -235,7 +233,7 @@ func (sp *serverlessProvider) BootUp(options Options) error { switch err { default: return err - case errProjectNotExist: + case serverless.ErrProjectNotExist: logger.Infof("Creating %s project: %q", settings.Type, settings.Name) config, err = sp.createProject(settings, options, config) if err != nil { @@ -364,7 +362,7 @@ func (sp *serverlessProvider) Status(options Options) ([]ServiceStatus, error) { } project, err := sp.currentProject(config) - if errors.Is(errProjectNotExist, err) { + if errors.Is(serverless.ErrProjectNotExist, err) { return nil, nil } if err != nil { From ff9cd38f3bac5e634dbbf8ef24342fcaa2b6b20a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 5 Sep 2023 11:40:13 +0200 Subject: [PATCH 48/62] Set allowed project types as a list --- internal/stack/serverless.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 445734fd38..6eeeaa3ad9 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/elastic/elastic-package/internal/common" "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/docker" "github.com/elastic/elastic-package/internal/elasticsearch" @@ -33,9 +34,9 @@ const ( ) var ( - allowedProjectTypes = map[string]struct{}{ - "security": {}, - "observability": {}, + allowedProjectTypes = []string{ + "security", + "observability", } ) @@ -222,8 +223,7 @@ func (sp *serverlessProvider) BootUp(options Options) error { return err } - _, ok := allowedProjectTypes[settings.Type] - if !ok { + if common.StringSliceContains(allowedProjectTypes, settings.Type) { return fmt.Errorf("serverless project type not supported: %s", settings.Type) } From 57e890f41abae9f0e332d5aacb463abd6cb8cd6d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 5 Sep 2023 12:01:01 +0200 Subject: [PATCH 49/62] Add new profile config to set elastic cloud host URL Added descriptions for new profile configuirations added --- internal/profile/_static/config.yml.example | 4 ++++ internal/stack/serverless.go | 12 +++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/internal/profile/_static/config.yml.example b/internal/profile/_static/config.yml.example index 14b3cf0687..ae35605f18 100644 --- a/internal/profile/_static/config.yml.example +++ b/internal/profile/_static/config.yml.example @@ -1,4 +1,8 @@ # Directory containing GeoIP databases for stacks managed by elastic-agent. # stack.geoip_dir: "/path/to/geoip_dir/" +# Severless project type to be created # stack.serverless.type: observability +# Region where the Serverless project is going to be created # stack.serverless.region: aws-us-east-1 +# Elastic cloud host URL +# stack.elastic_cloud.host diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 6eeeaa3ad9..ce99dad7e3 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -26,8 +26,9 @@ const ( paramServerlessProjectType = "serverless_project_type" paramServerlessFleetURL = "serverless_fleet_url" - configRegion = "stack.serverless.region" - configProjectType = "stack.serverless.type" + configRegion = "stack.serverless.region" + configProjectType = "stack.serverless.type" + configElasticCloudURL = "stack.elastic_agent.host" defaultRegion = "aws-us-east-1" defaultProjectType = "observability" @@ -202,7 +203,12 @@ func createProjectName(options Options) string { } func newServerlessProvider(profile *profile.Profile) (*serverlessProvider, error) { - client, err := serverless.NewClient() + host := profile.Config(configElasticCloudURL, "") + options := []serverless.ClientOption{} + if host != "" { + options = append(options, serverless.WithAddress(host)) + } + client, err := serverless.NewClient(options...) if err != nil { return nil, fmt.Errorf("can't create serverless provider: %w", err) } From b54e095266f62db6cb8a6fdf57d8d94b5b3356a9 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 5 Sep 2023 12:13:26 +0200 Subject: [PATCH 50/62] Use default host url is env. var is empty --- internal/serverless/client.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/internal/serverless/client.go b/internal/serverless/client.go index bece791a28..3c2c740a77 100644 --- a/internal/serverless/client.go +++ b/internal/serverless/client.go @@ -42,12 +42,10 @@ var ( func NewClient(opts ...ClientOption) (*Client, error) { hostEnvName := ServerlessHostvironmentVariable - host, ok := os.LookupEnv(hostEnvName) - if !ok { - host = defaultHostURL - } + host := os.Getenv(hostEnvName) if host == "" { - return nil, fmt.Errorf("unable to obtain value from %s environment variable", hostEnvName) + logger.Debugf("Using default host URL: %s", defaultHostURL) + host = defaultHostURL } apiKeyEnvName := ServerlessApiKeyEnvironmentVariable apiKey := os.Getenv(apiKeyEnvName) @@ -292,7 +290,7 @@ func (c *Client) GetProject(projectType, projectID string) (*Project, error) { } statusCode, respBody, err := c.get(ctx, resourcePath) if err != nil { - return nil, fmt.Errorf("error deleting project: %w", err) + return nil, fmt.Errorf("error getting project: %w", err) } if statusCode == http.StatusNotFound { From da1761a2ec53692b8548db0f22074ca0897f29a6 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 5 Sep 2023 12:32:39 +0200 Subject: [PATCH 51/62] Fix condition allowed project type --- internal/kibana/client.go | 4 ++-- internal/kibana/status.go | 25 +++++++------------------ internal/stack/serverless.go | 2 +- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/internal/kibana/client.go b/internal/kibana/client.go index 0f5ddaaf4d..6b9522c499 100644 --- a/internal/kibana/client.go +++ b/internal/kibana/client.go @@ -52,11 +52,11 @@ func NewClient(opts ...ClientOption) (*Client, error) { // Allow to initialize version from tests. var zeroVersion VersionInfo if c.semver == nil || c.versionInfo == zeroVersion { - v, err := c.requestVersion() + v, err := c.requestStatus() if err != nil { return nil, fmt.Errorf("failed to get Kibana version: %w", err) } - c.versionInfo = v + c.versionInfo = v.Version c.semver, err = semver.NewVersion(c.versionInfo.Number) if err != nil { diff --git a/internal/kibana/status.go b/internal/kibana/status.go index c9324f9107..adfef541a5 100644 --- a/internal/kibana/status.go +++ b/internal/kibana/status.go @@ -42,43 +42,32 @@ func (c *Client) Version() (VersionInfo, error) { return c.versionInfo, nil } -func (c *Client) requestVersion() (VersionInfo, error) { - var version VersionInfo +func (c *Client) requestStatus() (statusType, error) { + var status statusType statusCode, respBody, err := c.get(StatusAPI) if err != nil { - return version, fmt.Errorf("could not reach status endpoint: %w", err) + return status, fmt.Errorf("could not reach status endpoint: %w", err) } if statusCode != http.StatusOK { - return version, fmt.Errorf("could not get status data; API status code = %d; response body = %s", statusCode, respBody) + return status, fmt.Errorf("could not get status data; API status code = %d; response body = %s", statusCode, respBody) } - var status statusType err = json.Unmarshal(respBody, &status) if err != nil { - return version, fmt.Errorf("unmarshalling response failed (body: \n%s): %w", respBody, err) + return status, fmt.Errorf("unmarshalling response failed (body: \n%s): %w", respBody, err) } - return status.Version, nil + return status, nil } // CheckHealth checks the Kibana health func (c *Client) CheckHealth() error { - statusCode, respBody, err := c.get(StatusAPI) + status, err := c.requestStatus() if err != nil { return fmt.Errorf("could not reach status endpoint: %w", err) } - if statusCode != http.StatusOK { - return fmt.Errorf("could not get status data; API status code = %d; response body = %s", statusCode, respBody) - } - - var status statusType - err = json.Unmarshal(respBody, &status) - if err != nil { - return fmt.Errorf("unmarshalling response failed (body: \n%s): %w", respBody, err) - } - if status.Status.Overall.Level != "available" { return fmt.Errorf("kibana in unhealthy state: %s", status.Status.Overall.Level) } diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index ce99dad7e3..f60463e6d1 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -229,7 +229,7 @@ func (sp *serverlessProvider) BootUp(options Options) error { return err } - if common.StringSliceContains(allowedProjectTypes, settings.Type) { + if !common.StringSliceContains(allowedProjectTypes, settings.Type) { return fmt.Errorf("serverless project type not supported: %s", settings.Type) } From e95d1e5409445e1f1c898611836f8dfe9718e25c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 5 Sep 2023 16:08:58 +0200 Subject: [PATCH 52/62] Raise project not exist if any config parameters do not exist --- internal/stack/serverless.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index f60463e6d1..22017351c5 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -128,12 +128,12 @@ func (sp *serverlessProvider) deleteProject(project *serverless.Project, options func (sp *serverlessProvider) currentProject(config Config) (*serverless.Project, error) { projectID, found := config.Parameters[paramServerlessProjectID] if !found { - return nil, fmt.Errorf("mssing serverless project id") + return nil, serverless.ErrProjectNotExist } projectType, found := config.Parameters[paramServerlessProjectType] if !found { - return nil, fmt.Errorf("missing serverless project type") + return nil, serverless.ErrProjectNotExist } project, err := sp.client.GetProject(projectType, projectID) From 6bab96dd5c01ec4a987f3e2a6908f87405cb47c1 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 5 Sep 2023 17:04:46 +0200 Subject: [PATCH 53/62] Use same compose file as in compose provider (snapshot.yml) --- internal/stack/serverless.go | 2 +- internal/stack/serverlessresources.go | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 22017351c5..03e07e6fa2 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -283,7 +283,7 @@ func (sp *serverlessProvider) composeProjectName() string { } func (sp *serverlessProvider) localAgentComposeProject() (*compose.Project, error) { - composeFile := sp.profile.Path(profileStackPath, ServerlessComposeFile) + composeFile := sp.profile.Path(profileStackPath, SnapshotFile) return compose.NewProject(sp.composeProjectName(), composeFile) } diff --git a/internal/stack/serverlessresources.go b/internal/stack/serverlessresources.go index 3e01837d51..76c1630d3d 100644 --- a/internal/stack/serverlessresources.go +++ b/internal/stack/serverlessresources.go @@ -16,15 +16,10 @@ import ( "github.com/elastic/elastic-package/internal/profile" ) -const ( - // ServerlessComposeFile is the docker-compose snapshot.yml file name. - ServerlessComposeFile = "serverless-elastic-agent.yml" -) - var ( serverlessStackResources = []resource.Resource{ &resource.File{ - Path: ServerlessComposeFile, + Path: SnapshotFile, Content: staticSource.Template("_static/serverless-elastic-agent.yml.tmpl"), }, &resource.File{ From 1ad954eedc7aa78cccfb5b1783898d8df6336f6a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 5 Sep 2023 19:31:35 +0200 Subject: [PATCH 54/62] Retry docker-compose up once if elastic-agent failed --- internal/stack/serverless.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index 03e07e6fa2..ca2d46c5d7 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -305,7 +305,17 @@ func (sp *serverlessProvider) startLocalAgent(options Options, config Config) er err = project.Up(compose.CommandOptions{ExtraArgs: []string{"-d"}}) if err != nil { - return fmt.Errorf("failed to start local agent: %w", err) + // At least starting on 8.6.0, fleet-server may be reconfigured or + // restarted after being healthy. If elastic-agent tries to enroll at + // this moment, it fails inmediately, stopping and making `docker-compose up` + // to fail too. + // As a workaround, try to give another chance to docker-compose if only + // elastic-agent failed. + fmt.Println("Elastic Agent failed to start, trying again.") + err = project.Up(compose.CommandOptions{ExtraArgs: []string{"-d"}}) + if err != nil { + return fmt.Errorf("failed to start local agent: %w", err) + } } return nil From 74b90d294f7427eb8796adece7c8cffb2055ab5a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 6 Sep 2023 10:29:42 +0200 Subject: [PATCH 55/62] Add project type into status output --- internal/stack/serverless.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index ca2d46c5d7..caa224d51a 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -391,11 +391,12 @@ func (sp *serverlessProvider) Status(options Options) ([]ServiceStatus, error) { return nil, err } + serverlessVersion := fmt.Sprintf("serverless (%s)", project.Type) var serviceStatus []ServiceStatus for service, status := range projectServiceStatus { serviceStatus = append(serviceStatus, ServiceStatus{ Name: service, - Version: "serverless", + Version: serverlessVersion, Status: status, }) } From 5d582b5b9d13fe0d682b3e916dae0a3c8f30cc15 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 6 Sep 2023 17:54:10 +0200 Subject: [PATCH 56/62] Add example config value --- internal/profile/_static/config.yml.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/profile/_static/config.yml.example b/internal/profile/_static/config.yml.example index ae35605f18..10deb7ef04 100644 --- a/internal/profile/_static/config.yml.example +++ b/internal/profile/_static/config.yml.example @@ -5,4 +5,4 @@ # Region where the Serverless project is going to be created # stack.serverless.region: aws-us-east-1 # Elastic cloud host URL -# stack.elastic_cloud.host +# stack.elastic_cloud.host: https://cloud.elastic.co From acfce7d3815d8dc67fb67a6d9865c58c569f7992 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 6 Sep 2023 18:02:56 +0200 Subject: [PATCH 57/62] Fix typo in config.yml.example --- internal/profile/_static/config.yml.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/profile/_static/config.yml.example b/internal/profile/_static/config.yml.example index 10deb7ef04..68dee798d5 100644 --- a/internal/profile/_static/config.yml.example +++ b/internal/profile/_static/config.yml.example @@ -1,6 +1,6 @@ # Directory containing GeoIP databases for stacks managed by elastic-agent. # stack.geoip_dir: "/path/to/geoip_dir/" -# Severless project type to be created +# Serverless project type to be created # stack.serverless.type: observability # Region where the Serverless project is going to be created # stack.serverless.region: aws-us-east-1 From a24aa476671396eb8d200b00c5a3d4b213acfa6c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 7 Sep 2023 09:24:06 +0200 Subject: [PATCH 58/62] Apply suggestions from code review Co-authored-by: Jaime Soriano Pastor --- internal/profile/_static/config.yml.example | 10 +++++++--- internal/stack/serverless.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/profile/_static/config.yml.example b/internal/profile/_static/config.yml.example index 68dee798d5..c090af0b14 100644 --- a/internal/profile/_static/config.yml.example +++ b/internal/profile/_static/config.yml.example @@ -1,8 +1,12 @@ # Directory containing GeoIP databases for stacks managed by elastic-agent. # stack.geoip_dir: "/path/to/geoip_dir/" -# Serverless project type to be created +## Elastic Cloud +# Host URL +# stack.elastic_cloud.host: https://cloud.elastic.co + +## Serverless stack provider +# Project type # stack.serverless.type: observability # Region where the Serverless project is going to be created # stack.serverless.region: aws-us-east-1 -# Elastic cloud host URL -# stack.elastic_cloud.host: https://cloud.elastic.co + diff --git a/internal/stack/serverless.go b/internal/stack/serverless.go index caa224d51a..565ceeebb3 100644 --- a/internal/stack/serverless.go +++ b/internal/stack/serverless.go @@ -28,7 +28,7 @@ const ( configRegion = "stack.serverless.region" configProjectType = "stack.serverless.type" - configElasticCloudURL = "stack.elastic_agent.host" + configElasticCloudURL = "stack.elastic_cloud.host" defaultRegion = "aws-us-east-1" defaultProjectType = "observability" From 02648f8204b49bfee53efeae83a5541603112d18 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 7 Sep 2023 09:26:31 +0200 Subject: [PATCH 59/62] Rename variables and make it private --- internal/serverless/client.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/serverless/client.go b/internal/serverless/client.go index 3c2c740a77..eb0784332c 100644 --- a/internal/serverless/client.go +++ b/internal/serverless/client.go @@ -34,20 +34,20 @@ type Client struct { type ClientOption func(*Client) var ( - ServerlessApiKeyEnvironmentVariable = "EC_API_KEY" - ServerlessHostvironmentVariable = "EC_HOST" + elasticCloudApiKeyEnv = "EC_API_KEY" + elasticCloudEndpointEnv = "EC_HOST" ErrProjectNotExist = errors.New("project does not exist") ) func NewClient(opts ...ClientOption) (*Client, error) { - hostEnvName := ServerlessHostvironmentVariable + hostEnvName := elasticCloudEndpointEnv host := os.Getenv(hostEnvName) if host == "" { logger.Debugf("Using default host URL: %s", defaultHostURL) host = defaultHostURL } - apiKeyEnvName := ServerlessApiKeyEnvironmentVariable + apiKeyEnvName := elasticCloudApiKeyEnv apiKey := os.Getenv(apiKeyEnvName) if apiKey == "" { return nil, fmt.Errorf("unable to obtain value from %s environment variable", apiKeyEnvName) From 582bfa9848395178f189577eaf9d528a57aa61b4 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 7 Sep 2023 09:28:55 +0200 Subject: [PATCH 60/62] Show elastic cloud endpoint always as debug --- internal/serverless/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/serverless/client.go b/internal/serverless/client.go index eb0784332c..406fdf12e8 100644 --- a/internal/serverless/client.go +++ b/internal/serverless/client.go @@ -44,7 +44,6 @@ func NewClient(opts ...ClientOption) (*Client, error) { hostEnvName := elasticCloudEndpointEnv host := os.Getenv(hostEnvName) if host == "" { - logger.Debugf("Using default host URL: %s", defaultHostURL) host = defaultHostURL } apiKeyEnvName := elasticCloudApiKeyEnv @@ -52,6 +51,7 @@ func NewClient(opts ...ClientOption) (*Client, error) { if apiKey == "" { return nil, fmt.Errorf("unable to obtain value from %s environment variable", apiKeyEnvName) } + logger.Debugf("Using Elastic Cloud URL: %s", host) c := &Client{ host: host, apiKey: apiKey, From 6757dd3631133fa823fc6a313618fe8d0e968321 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 7 Sep 2023 10:02:20 +0200 Subject: [PATCH 61/62] Ensure preference between config and environment variables for host URL --- internal/profile/_static/config.yml.example | 2 +- internal/serverless/client.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/profile/_static/config.yml.example b/internal/profile/_static/config.yml.example index c090af0b14..e0f700f692 100644 --- a/internal/profile/_static/config.yml.example +++ b/internal/profile/_static/config.yml.example @@ -1,6 +1,6 @@ # Directory containing GeoIP databases for stacks managed by elastic-agent. # stack.geoip_dir: "/path/to/geoip_dir/" -## Elastic Cloud +## Elastic Cloud # Host URL # stack.elastic_cloud.host: https://cloud.elastic.co diff --git a/internal/serverless/client.go b/internal/serverless/client.go index 406fdf12e8..a865b8e0c7 100644 --- a/internal/serverless/client.go +++ b/internal/serverless/client.go @@ -41,25 +41,25 @@ var ( ) func NewClient(opts ...ClientOption) (*Client, error) { - hostEnvName := elasticCloudEndpointEnv - host := os.Getenv(hostEnvName) - if host == "" { - host = defaultHostURL - } apiKeyEnvName := elasticCloudApiKeyEnv apiKey := os.Getenv(apiKeyEnvName) if apiKey == "" { return nil, fmt.Errorf("unable to obtain value from %s environment variable", apiKeyEnvName) } - logger.Debugf("Using Elastic Cloud URL: %s", host) c := &Client{ - host: host, + host: defaultHostURL, apiKey: apiKey, } for _, opt := range opts { opt(c) } + hostEnvName := elasticCloudEndpointEnv + host := os.Getenv(hostEnvName) + if host != "" { + c.host = host + } + logger.Debugf("Using Elastic Cloud URL: %s", c.host) return c, nil } From e90f2832749cc088d2b3e978220b7de69232b1ab Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 7 Sep 2023 10:27:45 +0200 Subject: [PATCH 62/62] Remove variables --- internal/serverless/client.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/internal/serverless/client.go b/internal/serverless/client.go index a865b8e0c7..51ce89cbf7 100644 --- a/internal/serverless/client.go +++ b/internal/serverless/client.go @@ -41,10 +41,9 @@ var ( ) func NewClient(opts ...ClientOption) (*Client, error) { - apiKeyEnvName := elasticCloudApiKeyEnv - apiKey := os.Getenv(apiKeyEnvName) + apiKey := os.Getenv(elasticCloudApiKeyEnv) if apiKey == "" { - return nil, fmt.Errorf("unable to obtain value from %s environment variable", apiKeyEnvName) + return nil, fmt.Errorf("unable to obtain value from %s environment variable", elasticCloudApiKeyEnv) } c := &Client{ host: defaultHostURL, @@ -54,8 +53,7 @@ func NewClient(opts ...ClientOption) (*Client, error) { opt(c) } - hostEnvName := elasticCloudEndpointEnv - host := os.Getenv(hostEnvName) + host := os.Getenv(elasticCloudEndpointEnv) if host != "" { c.host = host }