Mattc/projectversioning (#48)

* Migrate to the new project versioning strategy

* Setting the versioning strategy count parameter

* Look up child steps as well

* Fixed comments

* Updated dependencies

* Updated dependencies

* Fixed donor package

* Reverting to old style of versioning, but leaving the new resources in place.

* Fix up variable scoping

* Added link to bug

* Skip the lookup if there is no deployment process

Author: mcasperson

cmd/internal/converters/deployment_process_converter_base.go

@@ -26,6 +26,7 @@ import (
 
 const octopusdeployProjectsDataType = "octopusdeploy_projects"
 const octopusdeployProjectResourceType = "octopusdeploy_project"
+const octopusdeployProjectVersioningStrategyResourceType = "octopusdeploy_project_versioning_strategy"
 
 type ProjectConverter struct {
 	Client                      client.OctopusClient
@@ -486,6 +487,9 @@ func (c *ProjectConverter) toHcl(project octopus.Project, recursive bool, lookup
 			return "", err
 		}
 
+		// We'll switch to the new versioning strategy once this bug is resolved:
+		// https://github.com/OctopusDeploy/terraform-provider-octopusdeploy/issues/55
+		//versioningStrategy, err := c.convertVersioningStrategyV2(project, projectName, dependencies)
 		versioningStrategy, err := c.convertVersioningStrategy(project)
 
 		if err != nil {
@@ -545,6 +549,11 @@ func (c *ProjectConverter) toHcl(project octopus.Project, recursive bool, lookup
 		if stateless {
 			c.writeData(file, "${var."+projectName+"_name}", projectName)
 			terraformResource.Count = strutil.StrPointer(thisResource.Count)
+
+			// This is used by the new versioning strategy resource
+			//if versioningStrategy != nil {
+			//	versioningStrategy.Count = strutil.StrPointer(thisResource.Count)
+			//}
 		}
 
 		block := gohcl.EncodeAsBlock(terraformResource, "resource")
@@ -597,6 +606,13 @@ func (c *ProjectConverter) toHcl(project octopus.Project, recursive bool, lookup
 		}
 
 		file.Body().AppendBlock(block)
+
+		// This is used by the new versioning strategy resource
+		//if versioningStrategy != nil {
+		//	versioningStrategyBlock := gohcl.EncodeAsBlock(versioningStrategy, "resource")
+		//	file.Body().AppendBlock(versioningStrategyBlock)
+		//}
+
 		return string(file.Bytes()), nil
 	}
 	dependencies.AddResource(thisResource)
@@ -977,6 +993,85 @@ func (c *ProjectConverter) convertUsernamePasswordGitPersistence(project octopus
 	}
 }
 
+// getDeploymentProcessStepId finds the internal ID of the action. This is despite the fact that the API calls the
+// parameter "DonorPackageStepId" - it is actually an Action ID, not a step ID.
+func (c *ProjectConverter) getDeploymentProcessStepId(project octopus.Project, dependencies *data.ResourceDetailsCollection) *string {
+	// The first action in a step is combined with the step, so here we lookup the "DeploymentProcesses/Steps" resource type
+	stepId := dependencies.GetResourceDependency("DeploymentProcesses/Steps",
+		project.Id+"/"+
+			strutil.EmptyIfNil(project.DeploymentProcessId)+"/"+
+			strutil.EmptyIfNil(project.VersioningStrategy.DonorPackageStepId))
+
+	// Second and subsequent actions are represented as "DeploymentProcesses/ChildSteps" resources, which we also need to check
+	actionId := dependencies.GetResourceDependency("DeploymentProcesses/ChildSteps",
+		project.Id+"/"+
+			strutil.EmptyIfNil(project.DeploymentProcessId)+"/"+
+			strutil.EmptyIfNil(project.VersioningStrategy.DonorPackageStepId))
+
+	return strutil.NilIfEmpty(strutil.DefaultIfEmpty(stepId, actionId))
+}
+
+func (c *ProjectConverter) convertDatabaseVersioningStrategyV2(project octopus.Project, projectName string, dependencies *data.ResourceDetailsCollection) (*terraform.TerraformProjectVersioningStrategy, error) {
+	versioningStrategyTerraformResource := terraform.TerraformProjectVersioningStrategy{
+		Type:               octopusdeployProjectVersioningStrategyResourceType,
+		Name:               projectName,
+		Count:              nil,
+		ProjectId:          "${" + octopusdeployProjectResourceType + "." + projectName + ".id}",
+		SpaceId:            strutil.InputIfEnabled(c.IncludeSpaceInPopulation, dependencies.GetResourceDependency("Spaces", project.SpaceId)),
+		DonorPackageStepId: c.getDeploymentProcessStepId(project, dependencies),
+		Template:           strutil.NilIfEmpty(project.VersioningStrategy.Template),
+	}
+
+	if project.VersioningStrategy.DonorPackage != nil {
+		versioningStrategyTerraformResource.DonorPackage = &terraform.TerraformProjectVersioningStrategyDonorPackage{
+			DeploymentAction: strutil.EmptyIfNil(project.VersioningStrategy.DonorPackage.DeploymentAction),
+			PackageReference: strutil.EmptyIfNil(project.VersioningStrategy.DonorPackage.PackageReference),
+		}
+	}
+
+	return &versioningStrategyTerraformResource, nil
+}
+
+func (c *ProjectConverter) convertCaCVersioningStrategyV2(project octopus.Project, projectName string, dependencies *data.ResourceDetailsCollection) (*terraform.TerraformProjectVersioningStrategy, error) {
+	deploymentSettings := octopus.ProjectCacDeploymentSettings{}
+	if _, err := c.Client.GetResource("Projects/"+project.Id+"/"+project.PersistenceSettings.DefaultBranch+"/DeploymentSettings", &deploymentSettings); err != nil {
+		return nil, err
+	}
+
+	versioningStrategyTerraformResource := terraform.TerraformProjectVersioningStrategy{
+		Type:               octopusdeployProjectVersioningStrategyResourceType,
+		Name:               projectName,
+		Count:              nil,
+		ProjectId:          "${" + octopusdeployProjectResourceType + "." + projectName + ".id}",
+		SpaceId:            strutil.InputIfEnabled(c.IncludeSpaceInPopulation, dependencies.GetResourceDependency("Spaces", project.SpaceId)),
+		DonorPackageStepId: c.getDeploymentProcessStepId(project, dependencies),
+		Template:           strutil.NilIfEmpty(deploymentSettings.VersioningStrategy.Template),
+	}
+
+	if deploymentSettings.VersioningStrategy.DonorPackage != nil {
+		versioningStrategyTerraformResource.DonorPackage = &terraform.TerraformProjectVersioningStrategyDonorPackage{
+			DeploymentAction: strutil.EmptyIfNil(deploymentSettings.VersioningStrategy.DonorPackage.DeploymentAction),
+			PackageReference: strutil.EmptyIfNil(deploymentSettings.VersioningStrategy.DonorPackage.PackageReference),
+		}
+	}
+
+	return &versioningStrategyTerraformResource, nil
+}
+
+func (c *ProjectConverter) convertVersioningStrategy(project octopus.Project) (*terraform.TerraformVersioningStrategy, error) {
+	if c.IgnoreCacManagedValues && project.HasCacConfigured() {
+		return nil, nil
+	}
+
+	// If CaC is enabled, the top level ProjectConnectivityPolicy settings are supplied by the API but ignored..
+	// The actual values come from branch specific settings.
+	if project.HasCacConfigured() {
+		return c.convertCaCVersioningStrategy(project)
+	}
+
+	return c.convertDatabaseVersioningStrategy(project)
+}
+
 func (c *ProjectConverter) convertDatabaseVersioningStrategy(project octopus.Project) (*terraform.TerraformVersioningStrategy, error) {
 	// Don't define a versioning strategy if it is not set
 	if project.VersioningStrategy.Template == "" {
@@ -1034,18 +1129,18 @@ func (c *ProjectConverter) convertCaCVersioningStrategy(project octopus.Project)
 	return &versioningStrategy, nil
 }
 
-func (c *ProjectConverter) convertVersioningStrategy(project octopus.Project) (*terraform.TerraformVersioningStrategy, error) {
+func (c *ProjectConverter) convertVersioningStrategyV2(project octopus.Project, projectName string, dependencies *data.ResourceDetailsCollection) (*terraform.TerraformProjectVersioningStrategy, error) {
 	if c.IgnoreCacManagedValues && project.HasCacConfigured() {
 		return nil, nil
 	}
 
-	// If CaC is enabled, the top level ProjectConnectivityPolicy settings are supplied by the API but ignored..
+	// If CaC is enabled, the top level ProjectConnectivityPolicy settings are supplied by the API but ignored.
 	// The actual values come from branch specific settings.
 	if project.HasCacConfigured() {
-		return c.convertCaCVersioningStrategy(project)
+		return c.convertCaCVersioningStrategyV2(project, projectName, dependencies)
 	}
 
-	return c.convertDatabaseVersioningStrategy(project)
+	return c.convertDatabaseVersioningStrategyV2(project, projectName, dependencies)
 }
 
 // exportChildDependencies exports those dependencies that are always required regardless of the recursive flag.

cmd/internal/converters/project_converter.go

@@ -1579,6 +1579,27 @@ func (c *VariableSetConverter) convertDisplaySettings(prompt octopus.Prompt) *te
 	return &display
 }
 
+// getDeploymentProcessStepId finds the internal ID of the action. This is despite the fact that the API calls the
+// parameter "DonorPackageStepId" - it is actually an Action ID, not a step ID.
+func (c *VariableSetConverter) getDeploymentProcessStepId(projectId string, deploymentProcessId string, actionIds []string, dependencies *data.ResourceDetailsCollection) []string {
+	// If there is no deployment process, there are no action IDs
+	if deploymentProcessId == "" {
+		return []string{}
+	}
+
+	return lo.Map(actionIds, func(item string, index int) string {
+		// The first action in a step is combined with the step, so here we lookup the "DeploymentProcesses/Steps" resource type
+		stepDependency := dependencies.GetResource("DeploymentProcesses/Steps",
+			projectId+"/"+deploymentProcessId+"/"+item)
+
+		// Second and subsequent actions are represented as "DeploymentProcesses/ChildSteps" resources, which we also need to check
+		actionDependency := dependencies.GetResource("DeploymentProcesses/ChildSteps",
+			projectId+"/"+deploymentProcessId+"/"+item)
+
+		return strutil.DefaultIfEmpty(stepDependency, actionDependency)
+	})
+}
+
 func (c *VariableSetConverter) convertScope(variable octopus.Variable, variableSet octopus.VariableSet, dependencies *data.ResourceDetailsCollection) (*terraform.TerraformProjectVariableScope, error) {
 	filteredEnvironments := c.EnvironmentFilter.FilterEnvironmentScope(variable.Scope.Environment)
 
@@ -1587,28 +1608,17 @@ func (c *VariableSetConverter) convertScope(variable octopus.Variable, variableS
 		zap.L().Warn("WARNING: Variable " + variable.Name + " removed all environment scopes.")
 	}
 
-	// Get a list of action IDs that includes the deployment process ID. This is because action IDs are
-	// not guaranteed to be unique and are saved in the dependencies with the deployment process ID as a prefix.
-	// If the variable set belongs to a library variable set, it can't be scoped to action, and so the profix
-	// won't be used.
-	actionPrefix, err := c.getDeploymentProcessFromVariable(variableSet)
+	deploymentProcessId, err := c.getDeploymentProcessFromVariable(variableSet)
 
 	if err != nil {
 		return nil, err
 	}
 
-	fixedActionIds := lo.Map(variable.Scope.Action, func(actionId string, index int) string {
-		return actionPrefix + "-" + actionId
-	})
-
-	// We should always find a prefix if there were any variable scopes. There may be edge cases
-	// where the action was deleted after the variable was created, for example a CaC project edited
-	// the deployment process but not the variable scopes.
-	if len(actionPrefix) == 0 && len(variable.Scope.Action) != 0 {
-		zap.L().Warn("WARNING: Variable " + variable.Name + " has action scopes but we did not find the deployment process. This means variables arr scoped to actions that do not exist.")
-	}
-
-	actions := dependencies.GetResources("Actions", fixedActionIds...)
+	// BUG: A octopusdeploy_process_child_step includes both the step and the first action. Hwoever, the ID is the step ID.
+	// There is no way to get the ID of the action. This means that any variables scoped to the first step in a process
+	// will be recreated with the step ID, and not the action ID, leading to a "Missing Resource" error in the UI.
+	// See https://github.com/OctopusDeploy/terraform-provider-octopusdeploy/issues/60
+	actions := c.getDeploymentProcessStepId(strutil.EmptyIfNil(variableSet.OwnerId), deploymentProcessId, variable.Scope.Action, dependencies)
 	channels := dependencies.GetResources("Channels", variable.Scope.Channel...)
 	environments := dependencies.GetResources("Environments", filteredEnvironments...)
 	machines := dependencies.GetResources("Machines", variable.Scope.Machine...)

cmd/internal/converters/variable_set_converter.go

@@ -1,10 +1,11 @@
 package data
 
 import (
+	"sync"
+
 	"github.com/OctopusSolutionsEngineering/OctopusTerraformExport/cmd/internal/strutil"
 	"github.com/samber/lo"
 	"go.uber.org/zap"
-	"sync"
 )
 
 type ToHcl func() (string, error)
@@ -29,7 +30,13 @@ type ResourceParameter struct {
 type ResourceDetails struct {
 	// Id is the octopus ID of the exported resource
 	Id string
-	// ParentId is an optional field that allows a resource to define its parenr.
+	// AlternateId is the alternate octopus ID of the exported resource.
+	// This can occur when a single Terraform resource represents multiple Octopus resources.
+	// An example is an octopusdeploy_process_step resource, which represents the combination of a step
+	// and the first action in the step. Sometimes we need to reference the step by its ID, and sometimes
+	// we need to reference the action by its ID.
+	AlternateId string
+	// ParentId is an optional field that allows a resource to define its parent.
 	// This is useful when establishing dependencies between Terraform resources where it is not easy to identify the
 	// individual Terraform resources that belong to a parent. For example, a channel must depend on the steps in a project
 	// because a channel references step packages by name, and thus do not establish a direct relationship that can be
@@ -161,7 +168,7 @@ func (c *ResourceDetailsCollection) GetResource(resourceType string, id string)
 	defer c.mu.Unlock()
 
 	for _, r := range c.Resources {
-		if r.Id == id && r.ResourceType == resourceType {
+		if (r.Id == id || r.AlternateId == id) && r.ResourceType == resourceType {
 			return r.Lookup
 		}
 	}
@@ -247,7 +254,7 @@ func (c *ResourceDetailsCollection) GetResourceDependency(resourceType string, i
 	defer c.mu.Unlock()
 
 	for _, r := range c.Resources {
-		if r.Id == id && r.ResourceType == resourceType {
+		if (r.Id == id || r.AlternateId == id) && r.ResourceType == resourceType {
 			// return the dependency field if it was defined, otherwise fall back to the lookup field
 			return strutil.DefaultIfEmpty(r.Dependency, r.Lookup)
 		}
@@ -258,6 +265,29 @@ func (c *ResourceDetailsCollection) GetResourceDependency(resourceType string, i
 	return ""
 }
 
+// GetResourceDependencyPointer returns the terraform references for a given resource type and id.
+// The returned string is used only for the depends_on field, as it may reference to a collection of resources
+// rather than a single ID.
+func (c *ResourceDetailsCollection) GetResourceDependencyPointer(resourceType string, id *string) *string {
+	if id == nil {
+		return nil
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	for _, r := range c.Resources {
+		if r.Id == *id && r.ResourceType == resourceType {
+			// return the dependency field if it was defined, otherwise fall back to the lookup field
+			return strutil.NilIfEmpty(strutil.DefaultIfEmpty(r.Dependency, r.Lookup))
+		}
+	}
+
+	zap.L().Error("Failed to resolve dependency " + *id + " of type " + resourceType)
+
+	return nil
+}
+
 // GetResourceDependencyFromParent returns the terraform references for a given resource type based on the parent ID.
 func (c *ResourceDetailsCollection) GetResourceDependencyFromParent(parentId string, resourceType string) []string {
 	c.mu.Lock()

cmd/internal/data/export_map.go

@@ -23,8 +23,19 @@ type TerraformProject struct {
 	GitLibraryPersistenceSettings          *TerraformGitLibraryPersistenceSettings          `hcl:"git_library_persistence_settings,block"`
 	GitAnonymousPersistenceSettings        *TerraformGitAnonymousPersistenceSettings        `hcl:"git_anonymous_persistence_settings,block"`
 	GitUsernamePasswordPersistenceSettings *TerraformGitUsernamePasswordPersistenceSettings `hcl:"git_username_password_persistence_settings,block"`
-	VersioningStrategy                     *TerraformVersioningStrategy                     `hcl:"versioning_strategy,block"`
 	Lifecycle                              *TerraformLifecycleMetaArgument                  `hcl:"lifecycle,block"`
+	VersioningStrategy                     *TerraformVersioningStrategy                     `hcl:"versioning_strategy,block"`
+}
+
+type TerraformVersioningStrategy struct {
+	Template           string                 `hcl:"template"`
+	DonorPackageStepId *string                `hcl:"donor_package_step_id"`
+	DonorPackage       *TerraformDonorPackage `hcl:"donor_package,block"`
+}
+
+type TerraformDonorPackage struct {
+	DeploymentAction *string `hcl:"deployment_action"`
+	PackageReference *string `hcl:"package_reference"`
 }
 
 func (t TerraformProject) HasCacConfigured() bool {
@@ -73,17 +84,6 @@ type TerraformGitUsernamePasswordPersistenceSettings struct {
 	ProtectedBranches string `hcl:"protected_branches"`
 }
 
-type TerraformVersioningStrategy struct {
-	Template           string                 `hcl:"template"`
-	DonorPackageStepId *string                `hcl:"donor_package_step_id"`
-	DonorPackage       *TerraformDonorPackage `hcl:"donor_package,block"`
-}
-
-type TerraformDonorPackage struct {
-	DeploymentAction *string `hcl:"deployment_action"`
-	PackageReference *string `hcl:"package_reference"`
-}
-
 type TerraformLifecycleMetaArgument struct {
 	CreateBeforeDestroy *bool                            `hcl:"create_before_destroy"`
 	IgnoreChanges       *[]string                        `hcl:"ignore_changes"`

cmd/internal/model/terraform/terraform_project.go

@@ -0,0 +1,17 @@
+package terraform
+
+type TerraformProjectVersioningStrategy struct {
+	Type               string                                          `hcl:"type,label"`
+	Name               string                                          `hcl:"name,label"`
+	Count              *string                                         `hcl:"count"`
+	ProjectId          string                                          `hcl:"project_id"`
+	SpaceId            *string                                         `hcl:"space_id"`
+	DonorPackage       *TerraformProjectVersioningStrategyDonorPackage `hcl:"donor_package,block"`
+	DonorPackageStepId *string                                         `hcl:"donor_package_step_id"`
+	Template           *string                                         `hcl:"template"`
+}
+
+type TerraformProjectVersioningStrategyDonorPackage struct {
+	DeploymentAction string `hcl:"deployment_action"`
+	PackageReference string `hcl:"package_reference"`
+}