chore: Adjust migration tests with SA and access token

.github/workflows/acceptance-tests-runner.yml

@@ -212,8 +212,10 @@ env:
   TF_LOG: ${{ vars.LOG_LEVEL }}
   ACCTEST_TIMEOUT: ${{ vars.ACCTEST_TIMEOUT }}
   # If the name (regex) of the test is set, only that test is run.
-  # Don't run migration tests if using Service Accounts because previous provider versions don't support SA yet.
   # Only Migration tests are run when a specific previous provider version is set.
+  # When using Service Account, migration tests are run in dedicated jobs using access tokens to avoid token creation limit issues.
+  # Currently dedicated jobs are: config_sa_mig (fast tests) and advanced_cluster_sa_mig (most important resource), more can be added as needed.
+  # This is because each migration test creates a new SA token in the test first step as the previous provider runs externally (equivalent to running a Terraform command).
   ACCTEST_REGEX_RUN: ${{ inputs.test_name || inputs.use_sa == true && '^TestAcc' || inputs.provider_version == '' && '^Test(Acc|Mig)' || '^TestMig' }}
   MONGODB_ATLAS_BASE_URL: ${{ inputs.mongodb_atlas_base_url }}
   MONGODB_REALM_BASE_URL: ${{ inputs.mongodb_realm_base_url }}
@@ -428,7 +430,7 @@ jobs:
 
   advanced_cluster_tpf_mig_from_sdkv2:
     needs: [ change-detection, get-provider-version ]
-    # Previous advanced_cluster versions don't support SA.
+    # advanced_cluster v1.x versions don't support SA.
     if: ${{ inputs.reduced_tests != true && inputs.use_sa != true && (needs.change-detection.outputs.advanced_cluster == 'true' || inputs.test_group == 'advanced_cluster') }}
     runs-on: ubuntu-latest
     permissions: {}
@@ -454,7 +456,7 @@ jobs:
 
   advanced_cluster_tpf_mig_from_tpf_preview:
     needs: [ change-detection, get-provider-version ]
-    # Previous advanced_cluster versions don't support SA.
+    # advanced_cluster v1.x versions don't support SA.
     if: ${{ inputs.reduced_tests != true && inputs.use_sa != true && (needs.change-detection.outputs.advanced_cluster == 'true' || inputs.test_group == 'advanced_cluster') }}
     runs-on: ubuntu-latest
     permissions: {}
@@ -479,6 +481,39 @@ jobs:
           ACCTEST_PACKAGES: ./internal/service/advancedcluster
         run: make testacc
 
+  advanced_cluster_sa_mig:
+    needs: [ change-detection, get-provider-version ]
+    if: ${{ inputs.use_sa == true && inputs.reduced_tests != true && (needs.change-detection.outputs.advanced_cluster == 'true' || inputs.test_group == 'advanced_cluster') }}
+    runs-on: ubuntu-latest
+    permissions: {}
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+      - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00
+        with:
+          go-version-file: 'go.mod'
+      - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd
+        with:
+          terraform_version: ${{ inputs.terraform_version }}
+          terraform_wrapper: false
+      - id: create-token
+        run: make access-token-create
+      - name: Migration Tests with Access Token
+        env:
+          MONGODB_ATLAS_PUBLIC_KEY: ""
+          MONGODB_ATLAS_PRIVATE_KEY: ""
+          MONGODB_ATLAS_CLIENT_ID: ""
+          MONGODB_ATLAS_CLIENT_SECRET: ""
+          MONGODB_ATLAS_ACCESS_TOKEN: ${{ steps.create-token.outputs.access_token }}
+          MONGODB_ATLAS_LAST_VERSION: ${{ needs.get-provider-version.outputs.provider_version }}
+          HTTP_MOCKER_CAPTURE: 'true'
+          ACCTEST_REGEX_RUN: '^TestMig'
+          ACCTEST_PACKAGES: ./internal/service/advancedcluster
+        run: make testacc
+      - name: Revoke token
+        run: make access-token-revoke token="${{ steps.create-token.outputs.access_token }}"
+
   assume_role:
     needs: [ change-detection, get-provider-version ]
     if: ${{ needs.change-detection.outputs.assume_role == 'true' || inputs.test_group == 'assume_role' }}
@@ -563,51 +598,26 @@ jobs:
           ACCTEST_REGEX_RUN: '^TestAccServiceAccount'
           ACCTEST_PACKAGES: ./internal/provider
         run: make testacc
-      - name: Generate OAuth2 Token
-        id: generate-token
-        shell: bash
+      - id: create-token
         env:
-          MONGODB_ATLAS_BASE_URL: ${{ inputs.mongodb_atlas_base_url }}
           MONGODB_ATLAS_CLIENT_ID: ${{ secrets.mongodb_atlas_client_id }}
-          MONGODB_ATLAS_CLIENT_SECRET: ${{ secrets.mongodb_atlas_client_secret }}
-        run: |
-          if ! ACCESS_TOKEN=$(make generate-oauth2-token); then
-            echo "Error: Failed to generate access token"
-            exit 1
-          fi
-          if [ -z "$ACCESS_TOKEN" ]; then
-            echo "Error: Generated access token is empty"
-            exit 1
-          fi
-          {
-            echo "access_token<<EOF"
-            echo "$ACCESS_TOKEN"
-            echo "EOF"
-          } >> "$GITHUB_OUTPUT"
+          MONGODB_ATLAS_CLIENT_SECRET: ${{ secrets.mongodb_atlas_client_secret }}        
+        run: make access-token-create
       - name: Acceptance Tests (Access Token)
         env:
           MONGODB_ATLAS_PUBLIC_KEY: ""
           MONGODB_ATLAS_PRIVATE_KEY: ""
           MONGODB_ATLAS_CLIENT_ID: ""
           MONGODB_ATLAS_CLIENT_SECRET: ""
-          MONGODB_ATLAS_ACCESS_TOKEN: ${{ steps.generate-token.outputs.access_token }}
-          MONGODB_ATLAS_LAST_VERSION: ${{ needs.get-provider-version.outputs.provider_version }}
+          MONGODB_ATLAS_ACCESS_TOKEN: ${{ steps.create-token.outputs.access_token }}
           ACCTEST_REGEX_RUN: '^TestAccAccessToken'
           ACCTEST_PACKAGES: ./internal/provider
         run: make testacc
-      - name: Acceptance Tests (Service Account smoke tests) # small selection of fast tests to run with SA
+      - name: Revoke token
         env:
-          MONGODB_ATLAS_PUBLIC_KEY: ""
-          MONGODB_ATLAS_PRIVATE_KEY: ""
           MONGODB_ATLAS_CLIENT_ID: ${{ secrets.mongodb_atlas_client_id }}
-          MONGODB_ATLAS_CLIENT_SECRET: ${{ secrets.mongodb_atlas_client_secret }}
-          MONGODB_ATLAS_LAST_VERSION: ${{ needs.get-provider-version.outputs.provider_version }}
-          ACCTEST_REGEX_RUN: '^TestAcc' # Don't run migration tests because previous provider versions don't support SA.
-          ACCTEST_PACKAGES: |
-            ./internal/service/alertconfiguration
-            ./internal/service/databaseuser
-            ./internal/service/maintenancewindow
-        run: make testacc
+          MONGODB_ATLAS_CLIENT_SECRET: ${{ secrets.mongodb_atlas_client_secret }}        
+        run: make access-token-revoke token="${{ steps.create-token.outputs.access_token }}"
 
   autogen_fast:
     needs: [change-detection, get-provider-version]
@@ -861,6 +871,65 @@ jobs:
             ./internal/service/thirdpartyintegration
         run: make testacc
 
+  config_sa_mig:
+    needs: [ change-detection, get-provider-version ]
+    if: ${{ inputs.use_sa == true && (needs.change-detection.outputs.config == 'true' || inputs.test_group == 'config') }}
+    runs-on: ubuntu-latest
+    permissions: {}
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+      - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00
+        with:
+          go-version-file: 'go.mod'
+      - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd
+        with:
+          terraform_version: ${{ inputs.terraform_version }}
+          terraform_wrapper: false
+      - id: create-token
+        run: make access-token-create
+      - name: Migration Tests with Access Token
+        env:
+          MONGODB_ATLAS_PUBLIC_KEY: ""
+          MONGODB_ATLAS_PRIVATE_KEY: ""
+          MONGODB_ATLAS_CLIENT_ID: ""
+          MONGODB_ATLAS_CLIENT_SECRET: ""
+          MONGODB_ATLAS_ACCESS_TOKEN: ${{ steps.create-token.outputs.access_token }}
+          MONGODB_ATLAS_PROJECT_OWNER_ID: ${{ inputs.mongodb_atlas_project_owner_id }}
+          MONGODB_ATLAS_USERNAME: ${{ vars.MONGODB_ATLAS_USERNAME }}
+          MONGODB_ATLAS_USERNAME_2: ${{ vars.MONGODB_ATLAS_USERNAME_2 }}
+          AZURE_ATLAS_APP_ID: ${{ inputs.azure_atlas_app_id }}
+          AZURE_SERVICE_PRINCIPAL_ID: ${{ inputs.azure_service_principal_id }}
+          AZURE_TENANT_ID: ${{ inputs.azure_tenant_id }}
+          AWS_REGION: ${{ vars.AWS_REGION_LOWERCASE }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.aws_access_key_id }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.aws_secret_access_key }}
+          AWS_S3_BUCKET: ${{ secrets.aws_s3_bucket_federation }}
+          MONGODB_ATLAS_LAST_VERSION: ${{ needs.get-provider-version.outputs.provider_version }}
+          ACCTEST_REGEX_RUN: '^TestMig'
+          ACCTEST_PACKAGES: |
+            ./internal/config
+            ./internal/service/alertconfiguration
+            ./internal/service/atlasuser
+            ./internal/service/cloudprovideraccess
+            ./internal/service/customdbrole
+            ./internal/service/customdnsconfigurationclusteraws
+            ./internal/service/databaseuser
+            ./internal/service/maintenancewindow
+            ./internal/service/organization
+            ./internal/service/orginvitation
+            ./internal/service/projectapikey
+            ./internal/service/apikeyprojectassignment
+            ./internal/service/apikey
+            ./internal/service/rolesorgid
+            ./internal/service/team
+            ./internal/service/teamprojectassignment
+            ./internal/service/thirdpartyintegration
+        run: make testacc
+      - name: Revoke token
+        run: make access-token-revoke token="${{ steps.create-token.outputs.access_token }}"
+
   encryption:
     needs: [ change-detection, get-provider-version ]
     concurrency:

Makefile

@@ -201,9 +201,13 @@ check-changelog-entry-file: ## Check a changelog entry file in a PR
 jira-release-version: ## Update Jira version in a release
 	go run ./tools/jira-release-version/*.go
 
-.PHONY: generate-oauth2-token
-generate-oauth2-token: ## Generate OAuth2 access token from Service Account credentials
-	@go run ./tools/generate-oauth2-token/*.go
+.PHONY: access-token-create
+access-token-create: ## Create a new OAuth2 access token from Service Account credentials
+	@go run ./tools/access-token/*.go create
+
+.PHONY: access-token-revoke
+access-token-revoke: ## Revoke an OAuth2 access token. Usage: make access-token-revoke token=<token>
+	@go run ./tools/access-token/*.go revoke $(token)
 
 .PHONY: enable-autogen
 enable-autogen: ## Enable use of autogen resources in the provider

internal/provider/provider_authentication_test.go

@@ -12,6 +12,7 @@ import (
 func TestAccSTSAssumeRole_basic(t *testing.T) {
 	acc.SkipInPAK(t, "skipping as this test is for AWS credentials only")
 	acc.SkipInSA(t, "skipping as this test is for AWS credentials only")
+	acc.SkipInAccessToken(t, "skipping as this test is for AWS credentials only")
 	var (
 		resourceName = "mongodbatlas_project.test"
 		orgID        = os.Getenv("MONGODB_ATLAS_ORG_ID")
@@ -44,6 +45,7 @@ func TestAccSTSAssumeRole_basic(t *testing.T) {
 
 func TestAccServiceAccount_basic(t *testing.T) {
 	acc.SkipInPAK(t, "skipping as this test is for SA only")
+	acc.SkipInAccessToken(t, "skipping as this test is for SA only")
 	var (
 		resourceName = "data.mongodbatlas_organization.test"
 		orgID        = os.Getenv("MONGODB_ATLAS_ORG_ID")

internal/service/project/resource_project_migration_test.go

@@ -154,6 +154,7 @@ func TestMigProject_withLimits(t *testing.T) {
 // based on bug report: https://github.com/mongodb/terraform-provider-mongodbatlas/issues/2263
 func TestMigGovProject_regionUsageRestrictionsDefault(t *testing.T) {
 	acc.SkipInSA(t, "SA not supported in Gov tests yet")
+	acc.SkipInAccessToken(t, "SA not supported in Gov tests yet")
 	var (
 		orgID       = os.Getenv("MONGODB_ATLAS_GOV_ORG_ID")
 		projectName = acc.RandomProjectName()

internal/service/project/resource_project_test.go

@@ -639,6 +639,7 @@ func TestAccProject_basic(t *testing.T) {
 
 func TestAccGovProject_withProjectOwner(t *testing.T) {
 	acc.SkipInSA(t, "SA not supported in Gov tests yet")
+	acc.SkipInAccessToken(t, "SA not supported in Gov tests yet")
 	var (
 		orgID          = os.Getenv("MONGODB_ATLAS_GOV_ORG_ID")
 		projectOwnerID = os.Getenv("MONGODB_ATLAS_GOV_PROJECT_OWNER_ID")

internal/testutil/acc/factory.go

@@ -61,6 +61,7 @@ func init() {
 		PrivateKey:   os.Getenv("MONGODB_ATLAS_PRIVATE_KEY"),
 		ClientID:     os.Getenv("MONGODB_ATLAS_CLIENT_ID"),
 		ClientSecret: os.Getenv("MONGODB_ATLAS_CLIENT_SECRET"),
+		AccessToken:  os.Getenv("MONGODB_ATLAS_ACCESS_TOKEN"),
 		BaseURL:      os.Getenv("MONGODB_ATLAS_BASE_URL"),
 		RealmBaseURL: os.Getenv("MONGODB_REALM_BASE_URL"),
 	}

internal/testutil/acc/pre_check.go

@@ -14,11 +14,21 @@ func PreCheckBasic(tb testing.TB) {
 	if os.Getenv("MONGODB_ATLAS_ORG_ID") == "" {
 		tb.Fatal("`MONGODB_ATLAS_ORG_ID` must be set for acceptance testing")
 	}
-	if HasPAKCreds() && HasSACreds() {
-		tb.Fatal("PAK and SA credentials are defined in this test but only one should be set.")
+	authCount := 0
+	if HasPAKCreds() {
+		authCount++
 	}
-	if !HasPAKCreds() && !HasSACreds() {
-		tb.Fatal("No credentials are defined in this test, PAK or SA credentials should be set.")
+	if HasSACreds() {
+		authCount++
+	}
+	if HasAccessToken() {
+		authCount++
+	}
+	if authCount > 1 {
+		tb.Fatal("Multiple credentials are set (PAK, SA, Access Token) but only one should be set.")
+	}
+	if authCount == 0 {
+		tb.Fatal("No credentials are set, one of PAK, SA, or Access Token should be set.")
 	}
 }
 

internal/testutil/acc/skip.go

@@ -40,6 +40,10 @@ func HasSACreds() bool {
 	return os.Getenv("MONGODB_ATLAS_CLIENT_ID") != "" || os.Getenv("MONGODB_ATLAS_CLIENT_SECRET") != ""
 }
 
+func HasAccessToken() bool {
+	return os.Getenv("MONGODB_ATLAS_ACCESS_TOKEN") != ""
+}
+
 func SkipInSA(tb testing.TB, description string) {
 	tb.Helper()
 	if HasSACreds() {
@@ -53,3 +57,10 @@ func SkipInPAK(tb testing.TB, description string) {
 		tb.Skip(description)
 	}
 }
+
+func SkipInAccessToken(tb testing.TB, description string) {
+	tb.Helper()
+	if HasAccessToken() {
+		tb.Skip(description)
+	}
+}