Periodically run exhaustive tests on artifacts prepared with DRA pipeline (#18803) (#18822)

* Add pipeline for running exhaustive tests against artifact prepared by DRA

This commit adds a new pipeline to test builds tracked in logstash-releases.yml. For
snapshots it runs daily, for released versions weekly.

* TEMP: Add POC artifact acceptance cells to exhaustive tests

* rename artifacts to match testing framework and test on x86 cells

* handle computing expected logstash version from artifacts

* remove POC cells and make codereview updates

(cherry picked from commit 5c73a8f)

Co-authored-by: Cas Donoghue <cas.donoghue@gmail.com>

Author: mergify[bot]

.buildkite/scripts/exhaustive-tests/generate-snapshot-release-steps.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+
+import os
+import requests
+from requests.adapters import HTTPAdapter, Retry
+import yaml
+
+YAML_HEADER = '# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json\n'
+VERSIONS_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/1.x/logstash-versions.yml"
+
+TEST_MATRIX = [
+    ("x86_64", "deb", "ubuntu-2204", "gcp", "n2-standard-4"),
+    ("x86_64", "rpm", "almalinux-8", "gcp", "n2-standard-4"),
+    ("arm64", "deb", "ubuntu-2204-aarch64", "aws", "m6g.4xlarge"),
+    ("arm64", "rpm", "almalinux-8-aarch64", "aws", "m6g.4xlarge"),
+]
+
+
+def fetch_logstash_versions():
+    session = requests.Session()
+    retries = Retry(total=5, backoff_factor=1, status_forcelist=[408, 502, 503, 504])
+    session.mount("https://", HTTPAdapter(max_retries=retries))
+    response = session.get(VERSIONS_URL, timeout=30)
+    response.raise_for_status()
+    return yaml.safe_load(response.text)
+
+
+def slugify_bk_key(key):
+    return key.replace(".", "-").replace(" ", "-").replace("/", "-").lower()
+
+
+def aws_agent(vm_name, instance_type):
+    return {
+        "provider": "aws",
+        "imagePrefix": f"platform-ingest-logstash-{vm_name}",
+        "instanceType": instance_type,
+        "diskSizeGb": 200,
+    }
+
+
+def gcp_agent(vm_name, instance_type):
+    return {
+        "provider": "gcp",
+        "imageProject": "elastic-images-prod",
+        "image": f"family/platform-ingest-logstash-multi-jdk-{vm_name}",
+        "machineType": instance_type,
+        "diskSizeGb": 200,
+        "diskType": "pd-ssd",
+    }
+
+
+def generate_test_step(version, artifact_type, arch, pkg_type, vm_name, provider, instance_type):
+    agent = aws_agent(vm_name, instance_type) if provider == "aws" else gcp_agent(vm_name, instance_type)
+    return {
+        "label": f"{version} / {arch} / {pkg_type}",
+        "key": slugify_bk_key(f"test-{version}-{arch}-{pkg_type}"),
+        "agents": agent,
+        "env": {
+            "LS_VERSION": version,
+            "ARTIFACT_TYPE": artifact_type,
+            "PKG_TYPE": pkg_type,
+            "ARCH": arch,
+        },
+        "command": ".buildkite/scripts/exhaustive-tests/run-acceptance-from-artifact.sh",
+        "retry": {"automatic": [{"limit": 3}]},
+    }
+
+
+if __name__ == "__main__":
+    artifact_type = os.getenv("ARTIFACT_TYPE", "snapshot")
+    versions_data = fetch_logstash_versions()
+    versions_dict = versions_data.get("snapshots" if artifact_type == "snapshot" else "releases", {})
+    versions = list(versions_dict.values())
+
+    steps = []
+    for version in versions:
+        for arch, pkg_type, vm_name, provider, instance_type in TEST_MATRIX:
+            steps.append(generate_test_step(version, artifact_type, arch, pkg_type, vm_name, provider, instance_type))
+
+    pipeline = {
+        "steps": [{
+            "group": f"Acceptance Tests ({artifact_type.capitalize()} Artifacts)",
+            "key": f"acceptance-{artifact_type}",
+            "steps": steps,
+        }]
+    }
+
+    print(YAML_HEADER + yaml.dump(pipeline, default_flow_style=False, sort_keys=False))

.buildkite/scripts/exhaustive-tests/run-acceptance-from-artifact.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# x86 runs on multi-jdk images, ARM runs on regular images
+if [[ "${ARCH}" == "x86_64" ]]; then
+  source .buildkite/scripts/common/vm-agent-multi-jdk.sh
+else
+  source .buildkite/scripts/common/vm-agent.sh
+fi
+
+# Get current source version - acceptance tests expect this version in filename
+SOURCE_VERSION=$(grep '^logstash:' versions.yml | awk '{print $2}')
+echo "Source version: ${SOURCE_VERSION}"
+
+# Map architecture to package-specific naming
+# deb uses: amd64/arm64, rpm uses: x86_64/aarch64
+case "${PKG_TYPE}" in
+  deb) [[ "${ARCH}" == "x86_64" ]] && PKG_ARCH="amd64" || PKG_ARCH="arm64" ;;
+  rpm) [[ "${ARCH}" == "x86_64" ]] && PKG_ARCH="x86_64" || PKG_ARCH="aarch64" ;;
+esac
+
+[[ "${ARTIFACT_TYPE}" == "snapshot" ]] && BASE_URL="https://snapshots.elastic.co/downloads/logstash" || BASE_URL="https://artifacts.elastic.co/downloads/logstash"
+
+ARTIFACT_DIR="${PWD}/acceptance-artifacts"
+mkdir -p "${ARTIFACT_DIR}"
+
+# Download artifact and rename to what acceptance tests expect
+ARTIFACT_URL="${BASE_URL}/logstash-${LS_VERSION}-${PKG_ARCH}.${PKG_TYPE}"
+EXPECTED_FILE="${ARTIFACT_DIR}/logstash-${SOURCE_VERSION}-SNAPSHOT-${PKG_ARCH}.${PKG_TYPE}"
+
+echo "--- Downloading Logstash ${LS_VERSION} ${PKG_ARCH} ${PKG_TYPE}"
+echo "URL: ${ARTIFACT_URL}"
+echo "Saving as: ${EXPECTED_FILE}"
+curl -fsSL -o "${EXPECTED_FILE}" "${ARTIFACT_URL}"
+
+echo "--- Downloaded artifacts"
+ls -la "${ARTIFACT_DIR}"
+
+echo "--- Running bootstrap"
+./gradlew clean bootstrap
+
+echo "--- Running acceptance tests"
+export LS_ARTIFACTS_PATH="${ARTIFACT_DIR}"
+echo "LS_ARTIFACTS_PATH=${LS_ARTIFACTS_PATH}"
+./gradlew runAcceptanceTests

.buildkite/snapshot_release_acceptance_pipeline.yml

@@ -0,0 +1,9 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+steps:
+  - label: "Generate acceptance test steps"
+    command: |
+      set -euo pipefail
+      python3 -m pip install pyyaml requests
+      python3 .buildkite/scripts/exhaustive-tests/generate-snapshot-release-steps.py > steps.yml
+      buildkite-agent pipeline upload < steps.yml

catalog-info.yaml

@@ -28,6 +28,7 @@ spec:
     - resource:buildkite-logstash-serverless-integration-testing
     - resource:logstash-snyk-report
     - resource:logstash-artifact-snyk-scan
+    - resource:logstash-artifacts-acceptance
     - resource:logstash-dra-snapshot-pipeline
     - resource:logstash-dra-staging-pipeline
     - resource:logstash-linux-jdk-matrix-pipeline
@@ -171,6 +172,56 @@ spec:
           cronline: "@daily"
           message: "Run the Logstash Artifacts Snyk report every day."
 
+# ***********************************
+# Declare artifacts acceptance test pipeline
+# ***********************************
+---
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: logstash-artifacts-acceptance
+  description: 'Acceptance tests for published snapshot and release artifacts'
+spec:
+  type: buildkite-pipeline
+  owner: group:logstash
+  system: platform-ingest
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: logstash-artifacts-acceptance
+    spec:
+      repository: elastic/logstash
+      pipeline_file: ".buildkite/snapshot_release_acceptance_pipeline.yml"
+      maximum_timeout_in_minutes: 120
+      provider_settings:
+        trigger_mode: none
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build'
+        SLACK_NOTIFICATIONS_ON_SUCCESS: 'false'
+        SLACK_NOTIFICATIONS_SKIP_FOR_RETRIES: 'true'
+      teams:
+        logstash:
+          access_level: MANAGE_BUILD_AND_READ
+        ingest-eng-prod:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
+      schedules:
+        Daily snapshot acceptance tests:
+          branch: main
+          cronline: "@daily"
+          message: "Test published snapshot artifacts"
+          env:
+            ARTIFACT_TYPE: "snapshot"
+        Weekly release acceptance tests:
+          branch: main
+          cronline: "@weekly"
+          message: "Test published release artifacts"
+          env:
+            ARTIFACT_TYPE: "release"
+
 # ***********************************
 # SECTION START: DRA pipelines
 # ***********************************

qa/acceptance/spec/shared_examples/cli/logstash/version.rb

@@ -18,6 +18,15 @@
 require_relative "../../../spec_helper"
 require "logstash/version"
 
+# When testing pre-built artifacts via the snapshot/release acceptance pipeline,
+# ARTIFACT_TYPE is set and LS_VERSION holds the artifact's version.
+# Derive the expected version from the artifact (stripping -SNAPSHOT suffix).
+EXPECTED_VERSION = if ENV['ARTIFACT_TYPE'] && ENV['LS_VERSION']
+                     ENV['LS_VERSION'].sub(/-SNAPSHOT$/, '')
+                   else
+                     LOGSTASH_VERSION
+                   end
+
 shared_examples "logstash version" do |logstash|
   describe "logstash --version" do
     before :all do
@@ -32,12 +41,12 @@
     context "on [#{logstash.human_name}]" do
       it "returns the right logstash version" do
         result = logstash.run_command_in_path("bin/logstash --version")
-        expect(result).to run_successfully_and_output(/#{LOGSTASH_VERSION}/)
+        expect(result).to run_successfully_and_output(/#{EXPECTED_VERSION}/)
       end
       context "when also using the --path.settings argument" do
         it "returns the right logstash version" do
           result = logstash.run_command_in_path("bin/logstash --path.settings=/etc/logstash --version")
-          expect(result).to run_successfully_and_output(/#{LOGSTASH_VERSION}/)
+          expect(result).to run_successfully_and_output(/#{EXPECTED_VERSION}/)
         end
       end
     end