diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f94c7468e..6ff20e94f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -156,6 +156,50 @@ jobs: if: always() run: bash ./scripts/dashboard/format_results.sh ${{job.status}} ${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}} ${{github.workspace}} + upgrade-tests: + name: Upgrade Tests + needs: build-unsigned-snapshot + runs-on: ubuntu-22.04 + strategy: + matrix: + container: + - image: "ubuntu" + version: "24.04" + - image: "redhatenterprise" + version: "9" + - image: "alpine" + version: "3.22" + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version-file: 'go.mod' + cache: false + - name: Download Packages + uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2 + with: + name: nginx-agent-unsigned-snapshots + path: build + + - name: Create Results Directory + run: mkdir -p ${{ github.workspace }}/test/dashboard/logs/${{ github.job }}/${{matrix.container.image}}-${{matrix.container.version}} + + - name: Start Promtail + uses: ./.github/actions/start-promtail + with: + loki_url: ${{ secrets.LOKI_DASHBOARD_URL }} + + - name: Run Upgrade Tests + run: | + go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }} + OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \ + make upgrade-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log + exit "${PIPESTATUS[0]}" + + - name: Format Results + if: always() + run: bash ./scripts/dashboard/format_results.sh ${{job.status}} ${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}} ${{github.workspace}} + official-oss-image-integration-tests: name: Integration Tests - Official OSS Images needs: build-unsigned-snapshot diff --git a/Makefile b/Makefile index 91d04147b..ed83c1333 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,7 @@ VERSION ?= $(shell git describe --match "v[0-9]*" --abbrev=0 --tags) ifeq ($(strip $(VERSION)),) VERSION := $(shell curl https://api.github.com/repos/nginx/agent/releases/latest -s | jq .name -r) endif + COMMIT = $(shell git rev-parse --short HEAD) DATE = $(shell date +%F_%H-%M-%S) LDFLAGS = "-s -w -X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.date=$(DATE)" @@ -166,6 +167,12 @@ integration-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) \ go test -v ./test/integration/installuninstall ./test/integration/managementplane ./test/integration/auxiliarycommandserver ./test/integration/nginxless +upgrade-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc + TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-repo" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \ + PACKAGES_REPO=$(OSS_PACKAGES_REPO) PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) \ + DOCKERFILE_PATH=$(DOCKERFILE_PATH) IMAGE_PATH=$(IMAGE_PATH) TAG=${IMAGE_TAG} OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) \ + go test -v ./test/integration/upgrade + official-image-integration-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} BUILD_TARGET="install" \ PACKAGES_REPO=$(OSS_PACKAGES_REPO) TAG=${TAG} PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) DOCKERFILE_PATH=$(OFFICIAL_IMAGE_DOCKERFILE_PATH) \ diff --git a/test/docker/nginx-oss/apk/Dockerfile b/test/docker/nginx-oss/apk/Dockerfile index 828c3c810..a8f6ede59 100644 --- a/test/docker/nginx-oss/apk/Dockerfile +++ b/test/docker/nginx-oss/apk/Dockerfile @@ -3,7 +3,6 @@ FROM ${BASE_IMAGE} as install-nginx LABEL maintainer="NGINX Docker Maintainers " ARG ENTRY_POINT -ARG PACKAGES_REPO WORKDIR /agent COPY ./build /agent/build @@ -33,12 +32,10 @@ RUN apk add --allow-untrusted /agent/build/${PACKAGE_NAME}.apk FROM install-nginx as install-agent-repo +ARG PACKAGES_REPO + # Setup nginx agent repository RUN curl -o /etc/apk/keys/nginx_signing.rsa.pub https://nginx.org/keys/nginx_signing.rsa.pub +RUN printf "http://${PACKAGES_REPO}/nginx-agent/alpine/v`grep -o -E '^[0-9]+\.[0-9]+' /etc/alpine-release`/main" | tee -a /etc/apk/repositories -RUN printf "%s%s%s\n" \ - "http://${PACKAGES_REPO}/nginx-agent/alpine/v" \ - `grep -o -E '^[0-9]+\.[0-9]+' /etc/alpine-release` \ - "/main" | tee -a /etc/apk/repositories - -RUN apk add nginx-agent@nginx-agent +RUN apk add nginx-agent diff --git a/test/integration/upgrade/configs/nginx-agent-v3-valid-config.conf b/test/integration/upgrade/configs/nginx-agent-v3-valid-config.conf new file mode 100644 index 000000000..215ed3890 --- /dev/null +++ b/test/integration/upgrade/configs/nginx-agent-v3-valid-config.conf @@ -0,0 +1,24 @@ +# +# /etc/nginx-agent/nginx-agent.conf +# +# Configuration file for NGINX Agent. +# + +log: + # set log level (error, warn, info, debug; default "info") + level: debug + # set log path. if empty, don't log to file. + path: /var/log/nginx-agent/ + +allowed_directories: + - /etc/nginx + - /usr/local/etc/nginx + - /usr/share/nginx/modules + - /var/run/nginx + - /var/log/nginx + +command: + server: + host: managementPlane + port: 9092 + type: grpc diff --git a/test/integration/upgrade/configs/nginx-agent.conf b/test/integration/upgrade/configs/nginx-agent.conf new file mode 100644 index 000000000..215ed3890 --- /dev/null +++ b/test/integration/upgrade/configs/nginx-agent.conf @@ -0,0 +1,24 @@ +# +# /etc/nginx-agent/nginx-agent.conf +# +# Configuration file for NGINX Agent. +# + +log: + # set log level (error, warn, info, debug; default "info") + level: debug + # set log path. if empty, don't log to file. + path: /var/log/nginx-agent/ + +allowed_directories: + - /etc/nginx + - /usr/local/etc/nginx + - /usr/share/nginx/modules + - /var/run/nginx + - /var/log/nginx + +command: + server: + host: managementPlane + port: 9092 + type: grpc diff --git a/test/integration/upgrade/configs/nginx-oss.conf b/test/integration/upgrade/configs/nginx-oss.conf new file mode 100644 index 000000000..8ba70f4cb --- /dev/null +++ b/test/integration/upgrade/configs/nginx-oss.conf @@ -0,0 +1,46 @@ +worker_processes 1; +error_log /var/log/nginx/error.log; +events { + worker_connections 1024; +} + +http { + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for" ' + '"$bytes_sent" "$request_length" "$request_time" ' + '"$gzip_ratio" $server_protocol '; + + access_log /var/log/nginx/access.log main; + + sendfile on; + keepalive_timeout 65; + + server { + listen 8080; + server_name localhost; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + ## + # Enable Metrics + ## + location /api { + stub_status; + allow 127.0.0.1; + deny all; + } + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + } +} diff --git a/test/integration/upgrade/configs/nginx-plus.conf b/test/integration/upgrade/configs/nginx-plus.conf new file mode 100644 index 000000000..1a7a0e83f --- /dev/null +++ b/test/integration/upgrade/configs/nginx-plus.conf @@ -0,0 +1,48 @@ +worker_processes 1; +error_log /var/log/nginx/error.log; + +events { + worker_connections 1024; +} + +http { + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for" ' + '"$bytes_sent" "$request_length" "$request_time" ' + '"$gzip_ratio" $server_protocol '; + + access_log /var/log/nginx/access.log main; + + sendfile on; + keepalive_timeout 65; + + server { + listen 8080; + server_name localhost; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + ## + # Enable Metrics + ## + location /api/ { + api write=on; + allow 127.0.0.1; + deny all; + status_zone my_location_zone1; + } + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + } +} diff --git a/test/integration/upgrade/upgrade_test.go b/test/integration/upgrade/upgrade_test.go new file mode 100644 index 000000000..1ab068502 --- /dev/null +++ b/test/integration/upgrade/upgrade_test.go @@ -0,0 +1,234 @@ +// Copyright (c) F5, Inc. +// +// This source code is licensed under the Apache License, Version 2.0 license found in the +// LICENSE file in the root directory of this source tree. + +package upgrade + +import ( + "bytes" + "context" + "io" + "log/slog" + "os" + "path" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/nginx/agent/v3/internal/model" + "github.com/nginx/agent/v3/test/helpers" + "github.com/nginx/agent/v3/test/integration/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" +) + +const ( + maxFileSize int64 = 70000000 + maxUpgradeTime = 30 * time.Second + agentBuildDir = "../agent/build" +) + +var ( + osRelease = os.Getenv("OS_RELEASE") + packageName = os.Getenv("PACKAGE_NAME") +) + +func Test_UpgradeFromV3(t *testing.T) { + ctx := context.Background() + + containerNetwork := utils.CreateContainerNetwork(ctx, t) + utils.SetupMockManagementPlaneGrpc(ctx, t, containerNetwork) + defer func(ctx context.Context) { + err := utils.MockManagementPlaneGrpcContainer.Terminate(ctx) + require.NoError(t, err) + }(ctx) + + testContainer, teardownTest := upgradeSetup(t, true, containerNetwork) + defer teardownTest(t) + + slog.Info("starting agent v3 upgrade tests") + + // get currently installed agent version + oldVersion := agentVersion(ctx, t, testContainer) + + // verify agent upgrade + verifyAgentUpgrade(ctx, t, testContainer) + + // verify version of agent + verifyAgentVersion(ctx, t, testContainer, oldVersion) + + // Verify Agent Package Path & get the path + verifyAgentPackageSize(t) + + // verify agent v3 config has not changed + validateAgentConfig(ctx, t, testContainer) + + // validate agent manifest file + expected := map[string]*model.ManifestFile{ + "/etc/nginx/nginx.conf": { + ManifestFileMeta: &model.ManifestFileMeta{ + Name: "/etc/nginx/nginx.conf", + Hash: "XEaOA4w+aT5fmNMISPwavBroLVYlkJf9sjKFTnWkTP8=", + Size: 1142, + Referenced: true, + }, + }, + } + utils.CheckManifestFile(t, testContainer, expected) + + slog.Info("finished agent v3 upgrade tests") +} + +func upgradeSetup(tb testing.TB, expectNoErrorsInLogs bool, + containerNetwork *testcontainers.DockerNetwork, +) (testcontainers.Container, func(tb testing.TB)) { + tb.Helper() + ctx := context.Background() + + params := &helpers.Parameters{ + NginxConfigPath: "./configs/nginx-oss.conf", + NginxAgentConfigPath: "./configs/nginx-agent.conf", + LogMessage: "nginx_pid", + } + + testContainer := helpers.StartContainer( + ctx, + tb, + containerNetwork, + params, + ) + + return testContainer, func(tb testing.TB) { + tb.Helper() + helpers.LogAndTerminateContainers( + ctx, + tb, + nil, + testContainer, + expectNoErrorsInLogs, + nil, + ) + } +} + +func verifyAgentPackageSize(tb testing.TB) string { + tb.Helper() + agentPkgPath, filePathErr := filepath.Abs("../../../build/") + require.NoError(tb, filePathErr, "Error finding local agent package build dir") + + localAgentPkg, packageErr := os.Stat(packagePath(agentPkgPath, osRelease)) + require.NoError(tb, packageErr, "Error accessing package at: "+agentPkgPath) + + // check if file size is less than 70MB + assert.Less(tb, localAgentPkg.Size(), maxFileSize) + + return packagePath(agentBuildDir, osRelease) +} + +func verifyAgentUpgrade(ctx context.Context, tb testing.TB, + testContainer testcontainers.Container, +) { + tb.Helper() + + // cmdOut for validating logs + _, upgradeTime := upgradeAgent(ctx, tb, testContainer) + + assert.LessOrEqual(tb, upgradeTime, maxUpgradeTime) + tb.Log("Upgrade time: ", upgradeTime) +} + +func upgradeAgent(ctx context.Context, tb testing.TB, testContainer testcontainers.Container, +) (io.Reader, time.Duration) { + tb.Helper() + + var upgradeCmd []string + + if strings.Contains(osRelease, "ubuntu") || strings.Contains(osRelease, "debian") { + upgradeCmd = []string{ + "apt-get", "install", "-y", "--only-upgrade", + "/agent/build/" + packageName + ".deb", "-o", "Dpkg::Options::=--force-confold", + } + } else if strings.Contains(osRelease, "alpine") { + upgradeCmd = []string{ + "apk", "add", "--allow-untrusted", "/agent/build/" + packageName + ".apk", + } + } else { + upgradeCmd = []string{"yum", "reinstall", "-y", "/agent/build/" + packageName + ".rpm"} + } + + start := time.Now() + + exitCode, cmdOut, err := testContainer.Exec(ctx, upgradeCmd) + require.NoError(tb, err) + + stdoutStderr, err := io.ReadAll(cmdOut) + require.NoError(tb, err) + + output := strings.TrimSpace(string(stdoutStderr)) + + require.NoError(tb, err) + assert.Equal(tb, 0, exitCode) + tb.Logf("Upgrade command output: %s", output) + + duration := time.Since(start) + + return cmdOut, duration +} + +func verifyAgentVersion(ctx context.Context, tb testing.TB, testContainer testcontainers.Container, oldVersion string) { + tb.Helper() + + newVersion := agentVersion(ctx, tb, testContainer) + assert.NotEqual(tb, oldVersion, newVersion) + tb.Logf("agent upgraded to version %s successfully", newVersion) +} + +func agentVersion(ctx context.Context, tb testing.TB, testContainer testcontainers.Container) string { + tb.Helper() + + cmd := []string{"nginx-agent", "--version"} + exitCode, cmdOut, err := testContainer.Exec(ctx, cmd) + require.NoError(tb, err) + assert.Equal(tb, 0, exitCode) + + stdoutStderr, err := io.ReadAll(cmdOut) + require.NoError(tb, err) + + output := strings.TrimSpace(string(stdoutStderr)) + + return output +} + +func packagePath(pkgDir, osReleaseContent string) string { + pkgPath := path.Join(pkgDir, packageName) + + if strings.Contains(osReleaseContent, "ubuntu") || strings.Contains(osReleaseContent, "Debian") { + return pkgPath + ".deb" + } else if strings.Contains(osReleaseContent, "alpine") { + return pkgPath + ".apk" + } + + return pkgPath + ".rpm" +} + +func validateAgentConfig(ctx context.Context, tb testing.TB, testContainer testcontainers.Container) { + tb.Helper() + + agentConfigContent, err := testContainer.CopyFileFromContainer(ctx, "/etc/nginx-agent/nginx-agent.conf") + require.NoError(tb, err) + + agentConfig, err := io.ReadAll(agentConfigContent) + require.NoError(tb, err) + + expectedConfig, err := os.ReadFile("./configs/nginx-agent-v3-valid-config.conf") + require.NoError(tb, err) + + expectedConfig = bytes.TrimSpace(expectedConfig) + agentConfig = bytes.TrimSpace(agentConfig) + + assert.Equal(tb, string(expectedConfig), string(agentConfig)) + tb.Log("agent config:", string(agentConfig)) +} diff --git a/test/integration/utils/config_apply_utils.go b/test/integration/utils/config_apply_utils.go index cc585a2d2..2a7e146c5 100644 --- a/test/integration/utils/config_apply_utils.go +++ b/test/integration/utils/config_apply_utils.go @@ -28,10 +28,11 @@ import ( ) const ( - RetryCount = 10 - RetryWaitTime = 5 * time.Second - RetryMaxWaitTime = 1 * time.Minute - permissions = 0o666 + RetryCount = 10 + RetryWaitTime = 5 * time.Second + RetryMaxWaitTime = 1 * time.Minute + permissions = 0o666 + manifestFileRetryWaitTime = 2 * time.Second ) var ( @@ -128,8 +129,21 @@ func CheckManifestFile(t *testing.T, container testcontainers.Container, expectedContent map[string]*model.ManifestFile, ) { t.Helper() - file, err := container.CopyFileFromContainer(t.Context(), "/var/lib/nginx-agent/manifest.json") + + var file io.ReadCloser + var err error + + retries := 5 + for i := range retries { + file, err = container.CopyFileFromContainer(t.Context(), "/var/lib/nginx-agent/manifest.json") + if err == nil { + break + } + t.Logf("Error copying manifest file, retry %d/%d: %v", i+1, retries, err) + time.Sleep(manifestFileRetryWaitTime) + } require.NoError(t, err) + fileContent, err := io.ReadAll(file) require.NoError(t, err) diff --git a/test/integration/utils/grpc_management_plane_utils.go b/test/integration/utils/grpc_management_plane_utils.go index e77c36a83..b8c82a9d5 100644 --- a/test/integration/utils/grpc_management_plane_utils.go +++ b/test/integration/utils/grpc_management_plane_utils.go @@ -100,8 +100,8 @@ func setupContainerEnvironment(ctx context.Context, tb testing.TB, nginxless, au tb.Helper() tb.Log("Running tests in a container environment") - containerNetwork := createContainerNetwork(ctx, tb) - setupMockManagementPlaneGrpc(ctx, tb, containerNetwork) + containerNetwork := CreateContainerNetwork(ctx, tb) + SetupMockManagementPlaneGrpc(ctx, tb, containerNetwork) if auxiliaryServer { setupAuxiliaryMockManagementPlaneGrpc(ctx, tb, containerNetwork) } @@ -118,8 +118,8 @@ func setupContainerEnvironment(ctx context.Context, tb testing.TB, nginxless, au } } -// createContainerNetwork creates and configures a container network. -func createContainerNetwork(ctx context.Context, tb testing.TB) *testcontainers.DockerNetwork { +// CreateContainerNetwork creates and configures a container network. +func CreateContainerNetwork(ctx context.Context, tb testing.TB) *testcontainers.DockerNetwork { tb.Helper() containerNetwork, err := network.New(ctx, network.WithAttachable()) require.NoError(tb, err) @@ -131,8 +131,8 @@ func createContainerNetwork(ctx context.Context, tb testing.TB) *testcontainers. return containerNetwork } -// setupMockManagementPlaneGrpc initializes the mock management plane gRPC container. -func setupMockManagementPlaneGrpc(ctx context.Context, tb testing.TB, containerNetwork *testcontainers.DockerNetwork) { +// SetupMockManagementPlaneGrpc initializes the mock management plane gRPC container. +func SetupMockManagementPlaneGrpc(ctx context.Context, tb testing.TB, containerNetwork *testcontainers.DockerNetwork) { tb.Helper() MockManagementPlaneGrpcContainer = helpers.StartMockManagementPlaneGrpcContainer(ctx, tb, containerNetwork) MockManagementPlaneGrpcAddress = "managementPlane:9092"