DO NOT MERGE Test aws integration on localstack API

bhapas · bhapas · commit c9496eb70993 · 2023-05-15T18:04:06.000+02:00
diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -18,37 +18,37 @@ STACK_COMMAND_TESTS=(
     test-stack-command-8x
 )
 
-for test in ${STACK_COMMAND_TESTS[@]}; do
-    echo "      - label: \":go: Running integration test: ${test}\""
-    echo "        command: ./.buildkite/scripts/integration_tests.sh -t ${test}"
-    echo "        agents:"
-    echo "          provider: \"gcp\""
-    echo "        artifact_paths:"
-    echo "          - build/elastic-stack-dump/stack/*/logs/*.log"
-    echo "          - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/**/*"
-    echo "          - build/elastic-stack-status/*/*"
-done
+# for test in ${STACK_COMMAND_TESTS[@]}; do
+#     echo "      - label: \":go: Running integration test: ${test}\""
+#     echo "        command: ./.buildkite/scripts/integration_tests.sh -t ${test}"
+#     echo "        agents:"
+#     echo "          provider: \"gcp\""
+#     echo "        artifact_paths:"
+#     echo "          - build/elastic-stack-dump/stack/*/logs/*.log"
+#     echo "          - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/**/*"
+#     echo "          - build/elastic-stack-status/*/*"
+# done
 
 CHECK_PACKAGES_TESTS=(
     test-check-packages-other
     test-check-packages-with-kind
     test-check-packages-with-custom-agent
     test-check-packages-benchmarks
 )
-for test in ${CHECK_PACKAGES_TESTS[@]}; do
-    echo "      - label: \":go: Running integration test: ${test}\""
-    echo "        command: ./.buildkite/scripts/integration_tests.sh -t ${test}"
-    echo "        agents:"
-    echo "          provider: \"gcp\""
-    echo "        artifact_paths:"
-    echo "          - build/test-results/*.xml"
-    echo "          - build/elastic-stack-dump/stack/check-*/logs/*.log"
-    echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/**/*"
-    echo "          - build/elastic-stack-status/*/*"
-    if [[ $test =~ with-kind$ ]]; then
-        echo "          - build/kubectl-dump.txt"
-    fi
-done
+# for test in ${CHECK_PACKAGES_TESTS[@]}; do
+#     echo "      - label: \":go: Running integration test: ${test}\""
+#     echo "        command: ./.buildkite/scripts/integration_tests.sh -t ${test}"
+#     echo "        agents:"
+#     echo "          provider: \"gcp\""
+#     echo "        artifact_paths:"
+#     echo "          - build/test-results/*.xml"
+#     echo "          - build/elastic-stack-dump/stack/check-*/logs/*.log"
+#     echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/**/*"
+#     echo "          - build/elastic-stack-status/*/*"
+#     if [[ $test =~ with-kind$ ]]; then
+#         echo "          - build/kubectl-dump.txt"
+#     fi
+# done
 
 pushd test/packages/parallel > /dev/null
 for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
diff --git a/Makefile b/Makefile
@@ -81,7 +81,7 @@ test-check-packages-benchmarks:
 	PACKAGE_TEST_TYPE=benchmarks ./scripts/test-check-packages.sh
 
 test-check-packages-parallel:
-	PACKAGE_TEST_TYPE=parallel ./scripts/test-check-packages.sh
+	PACKAGE_TEST_TYPE=parallel PACKAGE_UNDER_TEST=aws ./scripts/test-check-packages.sh
 
 test-check-packages-with-custom-agent:
 	PACKAGE_TEST_TYPE=with-custom-agent ./scripts/test-check-packages.sh
diff --git a/internal/testrunner/runners/system/servicedeployer/_static/localstack_deployer.yml b/internal/testrunner/runners/system/servicedeployer/_static/localstack_deployer.yml
@@ -0,0 +1,18 @@
+version: "3.8"
+services:
+  localstack:
+    container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}"
+    image: localstack/localstack
+    hostname: localstack
+    ports:
+      - "4566:4566"            # LocalStack Gateway
+    environment:
+      - SERVICES=sqs,sns
+      - DEBUG=1
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - HOST_TMP_FOLDER=${TMPDIR}
+      - HOSTNAME_EXTERNAL=localstack
+      - S3_HOSTNAME=localstack
+    volumes:
+      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
+      - "/var/run/docker.sock:/var/run/docker.sock"
diff --git a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh
@@ -21,6 +21,7 @@ trap cleanup EXIT INT TERM
 terraform init
 terraform plan
 terraform apply -auto-approve && touch /tmp/tf-applied
+terraform output -json
 
 echo "Terraform definitions applied."
 
diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go
@@ -24,10 +24,14 @@ import (
 const (
 	terraformDeployerDir        = "terraform"
 	terraformDeployerYml        = "terraform-deployer.yml"
+	localstackDeployerYml       = "localstack-deployer.yml"
 	terraformDeployerDockerfile = "Dockerfile"
 	terraformDeployerRun        = "run.sh"
 )
 
+//go:embed _static/localstack_deployer.yml
+var localstackDeployerYmlContent string
+
 //go:embed _static/terraform_deployer.yml
 var terraformDeployerYmlContent string
 
@@ -59,12 +63,21 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic
 	}
 
 	ymlPaths := []string{filepath.Join(configDir, terraformDeployerYml)}
+
+	localstackYmlPath := filepath.Join(configDir, localstackDeployerYml)
+	_, err = os.Stat(localstackYmlPath)
+	if err == nil {
+		ymlPaths = append(ymlPaths, localstackYmlPath)
+	}
+
 	envYmlPath := filepath.Join(tsd.definitionsDir, envYmlFile)
 	_, err = os.Stat(envYmlPath)
 	if err == nil {
 		ymlPaths = append(ymlPaths, envYmlPath)
 	}
 
+	logger.Debug("Print the yml Paths %s", ymlPaths)
+
 	tfEnvironment := tsd.buildTerraformExecutorEnvironment(inCtxt)
 
 	service := dockerComposeDeployedService{
@@ -130,6 +143,11 @@ func (tsd TerraformServiceDeployer) installDockerfile() (string, error) {
 	tfDir := filepath.Join(locationManager.DeployerDir(), terraformDeployerDir)
 
 	resources := []resource.Resource{
+		&resource.File{
+			Path:         localstackDeployerYml,
+			Content:      resource.FileContentLiteral(localstackDeployerYmlContent),
+			CreateParent: true,
+		},
 		&resource.File{
 			Path:         terraformDeployerYml,
 			Content:      resource.FileContentLiteral(terraformDeployerYmlContent),
diff --git a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/env.yml b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/env.yml
@@ -2,8 +2,4 @@ version: '2.3'
 services:
   terraform:
     environment:
-      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
-      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
-      - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}
-      - AWS_PROFILE=${AWS_PROFILE}
       - AWS_REGION=${AWS_REGION:-us-east-1}
diff --git a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf
@@ -3,31 +3,62 @@ variable "TEST_RUN_ID" {
 }
 
 provider "aws" {
-  default_tags {
-    tags = {
-      environment  = var.ENVIRONMENT
-      repo         = var.REPO
-      branch       = var.BRANCH
-      build        = var.BUILD_ID
-      created_date = var.CREATED_DATE
-    }
+  access_key                  = "test"
+  secret_key                  = "test"
+  region                      = "us-east-1"
+  s3_use_path_style           = true
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+
+  endpoints {
+    apigateway     = "http://localstack:4566"
+    apigatewayv2   = "http://localstack:4566"
+    cloudformation = "http://localstack:4566"
+    cloudwatch     = "http://localstack:4566"
+    dynamodb       = "http://localstack:4566"
+    ec2            = "http://localstack:4566"
+    es             = "http://localstack:4566"
+    elasticache    = "http://localstack:4566"
+    firehose       = "http://localstack:4566"
+    iam            = "http://localstack:4566"
+    kinesis        = "http://localstack:4566"
+    lambda         = "http://localstack:4566"
+    rds            = "http://localstack:4566"
+    redshift       = "http://localstack:4566"
+    route53        = "http://localstack:4566"
+    s3             = "http://localstack:4566"
+    secretsmanager = "http://localstack:4566"
+    ses            = "http://localstack:4566"
+    sns            = "http://localstack:4566"
+    sqs            = "http://localstack:4566"
+    ssm            = "http://localstack:4566"
+    stepfunctions  = "http://localstack:4566"
+    sts            = "http://localstack:4566"
   }
 }
 
-resource "aws_instance" "i" {
-  ami           = data.aws_ami.latest-amzn.id
-  monitoring = true
-  instance_type = "t1.micro"
+resource "aws_s3_bucket" "elastic_package_aws_test_bucket" {
+  bucket = "my-tf-test-bucket"
+
   tags = {
-    Name = "elastic-package-test-${var.TEST_RUN_ID}"
+    environment  = var.ENVIRONMENT
+    repo         = var.REPO
+    branch       = var.BRANCH
+    build        = var.BUILD_ID
+    created_date = var.CREATED_DATE
   }
 }
 
-data "aws_ami" "latest-amzn" {
-  most_recent = true
-  owners = [ "amazon" ] # AWS
-  filter {
-    name   = "name"
-    values = ["amzn2-ami-minimal-hvm-*-ebs"]
+resource "aws_instance" "elastic_package_aws_test_ec2" {
+  ami           = "ami-0d57c0143330e1fa7"
+  instance_type = "t2.micro"
+
+  tags = {
+    environment  = var.ENVIRONMENT
+    repo         = var.REPO
+    branch       = var.BRANCH
+    build        = var.BUILD_ID
+    created_date = var.CREATED_DATE
   }
 }
diff --git a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/variables.tf b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/variables.tf
@@ -1,16 +1,16 @@
 variable "BRANCH" {
   description = "Branch name or pull request for tagging purposes"
-  default = "unknown-branch"
+  default     = "unknown-branch"
 }
 
 variable "BUILD_ID" {
   description = "Build ID in the CI for tagging purposes"
-  default = "unknown-build"
+  default     = "unknown-build"
 }
 
 variable "CREATED_DATE" {
   description = "Creation date in epoch time for tagging purposes"
-  default = "unknown-date"
+  default     = "unknown-date"
 }
 
 variable "ENVIRONMENT" {
diff --git a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml
@@ -5,8 +5,8 @@ vars:
   session_token: '{{AWS_SESSION_TOKEN}}'
 data_stream:
   vars:
-    period: 5m
-    latency: 10m
+    period: 1m
+    latency: 1m
     tags_filter: |-
       - key: Name
         value: "elastic-package-test-{{TEST_RUN_ID}}"

Original file line number	Diff line number	Diff line change
`@@ -1,16 +1,16 @@`
`1`	`1`	`variable "BRANCH" {`
`2`	`2`	`description = "Branch name or pull request for tagging purposes"`
`3`		`- default = "unknown-branch"`
	`3`	`+ default = "unknown-branch"`
`4`	`4`	`}`
`5`	`5`
`6`	`6`	`variable "BUILD_ID" {`
`7`	`7`	`description = "Build ID in the CI for tagging purposes"`
`8`		`- default = "unknown-build"`
	`8`	`+ default = "unknown-build"`
`9`	`9`	`}`
`10`	`10`
`11`	`11`	`variable "CREATED_DATE" {`
`12`	`12`	`description = "Creation date in epoch time for tagging purposes"`
`13`		`- default = "unknown-date"`
	`13`	`+ default = "unknown-date"`
`14`	`14`	`}`
`15`	`15`
`16`	`16`	`variable "ENVIRONMENT" {`