diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 56e9a55b22..c1dd00d36e 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -8,6 +8,7 @@ * Added output attribute `endpoint_url` in `databricks_model_serving` ([#4877](https://github.com/databricks/terraform-provider-databricks/pull/4877)). * Deprecate `egg` library type in `databricks_cluster`, `databricks_job`, and `databricks_library` ([#4881](https://github.com/databricks/terraform-provider-databricks/pull/4881)). +* Added `databricks_groups` data source to retrieve multiple groups matching a filter criteria ([#4864](https://github.com/databricks/terraform-provider-databricks/pull/4864)). ### Bug Fixes diff --git a/docs/data-sources/groups.md b/docs/data-sources/groups.md new file mode 100644 index 0000000000..7b191d0b21 --- /dev/null +++ b/docs/data-sources/groups.md @@ -0,0 +1,110 @@ +--- +subcategory: "Security" +--- +# databricks_groups Data Source + +Retrieves information about multiple [databricks_group](../resources/group.md) objects matching a filter criteria. This data source allows you to search for groups using SCIM filter expressions and returns detailed information about matching groups including their members, entitlements and instance profiles. + +-> This data source can be used with an account or workspace-level provider. + +## Example Usage + +Retrieve all groups with "admin" in the name: + +```hcl +data "databricks_groups" "admin_groups" { + filter = "displayName co \"admin\"" +} + +resource "databricks_permissions" "cluster_permissions" { + cluster_id = databricks_cluster.example.id + + dynamic "access_control" { + for_each = data.databricks_groups.admin_groups.groups + content { + group_name = access_control.value.display_name + permission_level = "CAN_MANAGE" + } + } +} +``` + +Find groups by exact display name: + +```hcl +data "databricks_groups" "specific_teams" { + filter = "displayName eq \"team-data\" or displayName eq \"team-ml\"" +} + +locals { + team_principals = [for group in data.databricks_groups.specific_teams.groups : group.acl_principal_id] +} +``` + +Find all groups without a filter (retrieve all groups): + +```hcl +data "databricks_groups" "all_groups" { + filter = "" +} + +output "total_groups" { + value = length(data.databricks_groups.all_groups.groups) +} +``` + +## Argument Reference + +Data source allows you to filter groups by the following attributes: + +* `filter` - (Optional) SCIM filter expression to match groups. Uses the same filter syntax as the Databricks SCIM API. Common examples: + * `displayName co "admin"` - Groups containing "admin" in the display name + * `displayName eq "admins"` - Groups with exact display name "admins" + * `displayName sw "team-"` - Groups whose display name starts with "team-" + * Empty string or omitted - Returns all groups + +## Attribute Reference + +Data source exposes the following attributes: + +* `groups` - List of group objects matching the filter criteria. Each group object contains: + * `display_name` - Display name of the group. + * `external_id` - ID of the group in an external identity provider. + * `acl_principal_id` - Identifier for use in [databricks_access_control_rule_set](../resources/access_control_rule_set.md), formatted as `groups/{display_name}`. + * `members` - Set of all group member identifiers (users, service principals, and child groups combined). + * `users` - Set of [databricks_user](../resources/user.md) identifiers that are members of this group. + * `service_principals` - Set of [databricks_service_principal](../resources/service_principal.md) identifiers that are members of this group. + * `child_groups` - Set of [databricks_group](../resources/group.md) identifiers that are child groups of this group. + * `groups` - Set of parent [databricks_group](../resources/group.md) identifiers that this group belongs to. + * `instance_profiles` - Set of [instance profile](../resources/instance_profile.md) ARNs associated with this group. + +-> **Note**: The `groups` attribute returns a list sorted by `display_name` for consistent ordering. All member sets within each group are also sorted alphabetically. + +## SCIM Filter Syntax + +The `filter` parameter supports SCIM 2.0 filter expressions. Common operators include: + +* `co` - Contains (case insensitive) +* `eq` - Equals (case sensitive) +* `ne` - Not equals +* `sw` - Starts with +* `ew` - Ends with +* `and` - Logical AND +* `or` - Logical OR + +Examples: +* `displayName co "admin"` - Contains "admin" +* `displayName sw "team-" and displayName ew "-prod"` - Starts with "team-" and ends with "-prod" +* `displayName eq "admins" or displayName eq "developers"` - Exact match for either name + +## Related Resources + +The following resources are used in the same context: + +* [End to end workspace management](../guides/workspace-management.md) guide +* [databricks_group](group.md) to retrieve information about a single group +* [databricks_group](../resources/group.md) to manage groups in Databricks workspace +* [databricks_group_member](../resources/group_member.md) to manage group membership +* [databricks_permissions](../resources/permissions.md) to manage [access control](https://docs.databricks.com/security/access-control/index.html) in Databricks workspace +* [databricks_user](../resources/user.md) to manage users that can be added to groups +* [databricks_service_principal](../resources/service_principal.md) to manage service principals that can be added to groups \ No newline at end of file diff --git a/internal/providers/sdkv2/sdkv2.go b/internal/providers/sdkv2/sdkv2.go index bda107328e..380836d2f4 100644 --- a/internal/providers/sdkv2/sdkv2.go +++ b/internal/providers/sdkv2/sdkv2.go @@ -127,6 +127,7 @@ func DatabricksProvider(opts ...SdkV2ProviderOption) *schema.Provider { "databricks_external_location": catalog.DataSourceExternalLocation().ToResource(), "databricks_external_locations": catalog.DataSourceExternalLocations().ToResource(), "databricks_group": scim.DataSourceGroup().ToResource(), + "databricks_groups": scim.DataSourceGroups().ToResource(), "databricks_instance_pool": pools.DataSourceInstancePool().ToResource(), "databricks_instance_profiles": aws.DataSourceInstanceProfiles().ToResource(), "databricks_jobs": jobs.DataSourceJobs().ToResource(), diff --git a/scim/data_groups.go b/scim/data_groups.go new file mode 100644 index 0000000000..53e8ab8f20 --- /dev/null +++ b/scim/data_groups.go @@ -0,0 +1,94 @@ +package scim + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/databricks/terraform-provider-databricks/common" +) + +// DataSourceGroups returns information about groups that match the specified filter +func DataSourceGroups() common.Resource { + type groupEntity struct { + DisplayName string `json:"display_name"` + Recursive bool `json:"recursive,omitempty"` + Members []string `json:"members,omitempty" tf:"slice_set,computed"` + Users []string `json:"users,omitempty" tf:"slice_set,computed"` + ServicePrincipals []string `json:"service_principals,omitempty" tf:"slice_set,computed"` + ChildGroups []string `json:"child_groups,omitempty" tf:"slice_set,computed"` + Groups []string `json:"groups,omitempty" tf:"slice_set,computed"` + InstanceProfiles []string `json:"instance_profiles,omitempty" tf:"slice_set,computed"` + ExternalID string `json:"external_id,omitempty" tf:"computed"` + AclPrincipalID string `json:"acl_principal_id,omitempty" tf:"computed"` + } + + type groupsData struct { + Filter string `json:"filter,omitempty"` + Groups []groupEntity `json:"groups,omitempty" tf:"computed"` + } + + return common.DataResource(groupsData{}, func(ctx context.Context, e any, c *common.DatabricksClient) error { + response := e.(*groupsData) + groupsAPI := NewGroupsAPI(ctx, c) + + groupList, err := groupsAPI.Filter(response.Filter) + if err != nil { + return err + } + + // Initialize empty groups slice to ensure we return empty array when no matches + response.Groups = []groupEntity{} + + for _, group := range groupList.Resources { + // Convert each Group to the expected entity shape + entity := groupEntity{ + DisplayName: group.DisplayName, + ExternalID: group.ExternalID, + AclPrincipalID: fmt.Sprintf("groups/%s", group.DisplayName), + } + + // Process members to extract different types + for _, member := range group.Members { + entity.Members = append(entity.Members, member.Value) + if strings.HasPrefix(member.Ref, "Users/") { + entity.Users = append(entity.Users, member.Value) + } + if strings.HasPrefix(member.Ref, "Groups/") { + entity.ChildGroups = append(entity.ChildGroups, member.Value) + } + if strings.HasPrefix(member.Ref, "ServicePrincipals/") { + entity.ServicePrincipals = append(entity.ServicePrincipals, member.Value) + } + } + + // Process roles (instance profiles) + for _, role := range group.Roles { + entity.InstanceProfiles = append(entity.InstanceProfiles, role.Value) + } + + // Process parent groups + for _, parentGroup := range group.Groups { + entity.Groups = append(entity.Groups, parentGroup.Value) + } + + // Sort all slices for consistent output + sort.Strings(entity.Members) + sort.Strings(entity.Users) + sort.Strings(entity.ServicePrincipals) + sort.Strings(entity.ChildGroups) + sort.Strings(entity.Groups) + sort.Strings(entity.InstanceProfiles) + + response.Groups = append(response.Groups, entity) + } + + // Sort groups by display name for consistent output + sort.Slice(response.Groups, func(i, j int) bool { + return response.Groups[i].DisplayName < response.Groups[j].DisplayName + }) + + return nil + }) +} diff --git a/scim/data_groups_acc_test.go b/scim/data_groups_acc_test.go new file mode 100644 index 0000000000..19a05e396c --- /dev/null +++ b/scim/data_groups_acc_test.go @@ -0,0 +1,146 @@ +package scim_test + +import ( + "testing" + + "github.com/databricks/terraform-provider-databricks/internal/acceptance" + "github.com/hashicorp/terraform-plugin-testing/terraform" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const groupsDataSourceTemplate = ` +resource "databricks_user" "test_user" { + user_name = "tf-groups-user-{var.RANDOM}@example.com" +} + +resource "databricks_service_principal" "test_sp" { + application_id = "{var.RANDOM_UUID}" + display_name = "tf-groups-sp-{var.RANDOM}" + force = true +} + +resource "databricks_group" "test_group_1" { + display_name = "tf-groups-test-{var.RANDOM}-alpha" +} + +resource "databricks_group" "test_group_2" { + display_name = "tf-groups-test-{var.RANDOM}-beta" +} + +resource "databricks_group" "other_group" { + display_name = "tf-other-{var.RANDOM}" +} + +resource "databricks_group_member" "member_1_user" { + group_id = databricks_group.test_group_1.id + member_id = databricks_user.test_user.id +} + +resource "databricks_group_member" "member_1_sp" { + group_id = databricks_group.test_group_1.id + member_id = databricks_service_principal.test_sp.id +} + +resource "databricks_group_member" "member_2_group" { + group_id = databricks_group.test_group_2.id + member_id = databricks_group.test_group_1.id +} + +data "databricks_groups" "test_filter" { + filter = "displayName co \"tf-groups-test-{var.RANDOM}\"" + depends_on = [ + databricks_group_member.member_1_user, + databricks_group_member.member_1_sp, + databricks_group_member.member_2_group + ] +} + +data "databricks_groups" "exact_match" { + filter = "displayName eq \"${databricks_group.test_group_1.display_name}\"" + depends_on = [ + databricks_group_member.member_1_user, + databricks_group_member.member_1_sp + ] +} + +data "databricks_groups" "empty_filter" { + filter = "displayName co \"nonexistent-group-{var.RANDOM}\"" +}` + +func checkGroupsDataSourcePopulated(t *testing.T) func(s *terraform.State) error { + return func(s *terraform.State) error { + // Check the filtered data source that should return 2 groups + r, ok := s.Modules[0].Resources["data.databricks_groups.test_filter"] + require.True(t, ok, "data.databricks_groups.test_filter has to be there") + attr := r.Primary.Attributes + + // Should find 2 groups matching the filter + assert.Equal(t, "2", attr["groups.#"]) + + // Groups should be sorted alphabetically (alpha before beta) + group1DisplayName := attr["groups.0.display_name"] + group2DisplayName := attr["groups.1.display_name"] + + require.Contains(t, group1DisplayName, "alpha", "First group should be alpha (alphabetically first)") + require.Contains(t, group2DisplayName, "beta", "Second group should be beta (alphabetically second)") + + // Verify group 1 (alpha) has the expected members + assert.NotEmpty(t, attr["groups.0.users.0"], "Group 1 should have users") + assert.NotEmpty(t, attr["groups.0.service_principals.0"], "Group 1 should have service principals") + assert.Contains(t, attr["groups.0.acl_principal_id"], "groups/", "ACL principal ID should have groups/ prefix") + + // Verify group 2 (beta) has the expected child group + assert.NotEmpty(t, attr["groups.1.child_groups.0"], "Group 2 should have child groups") + assert.Contains(t, attr["groups.1.acl_principal_id"], "groups/", "ACL principal ID should have groups/ prefix") + + // Check the exact match data source that should return 1 group + exactMatch, ok := s.Modules[0].Resources["data.databricks_groups.exact_match"] + require.True(t, ok, "data.databricks_groups.exact_match has to be there") + exactAttr := exactMatch.Primary.Attributes + + assert.Equal(t, "1", exactAttr["groups.#"]) + assert.Contains(t, exactAttr["groups.0.display_name"], "alpha", "Exact match should return the alpha group") + + // Check empty result data source + emptyResult, ok := s.Modules[0].Resources["data.databricks_groups.empty_filter"] + require.True(t, ok, "data.databricks_groups.empty_filter has to be there") + emptyAttr := emptyResult.Primary.Attributes + + assert.Equal(t, "0", emptyAttr["groups.#"], "Filter matching no groups should return empty list") + + return nil + } +} + +func TestMwsAccGroupsDataSource(t *testing.T) { + acceptance.GetEnvOrSkipTest(t, "ARM_CLIENT_ID") + acceptance.AccountLevel(t, acceptance.Step{ + Template: groupsDataSourceTemplate, + Check: checkGroupsDataSourcePopulated(t), + }) +} + +func TestAccGroupsDataSource(t *testing.T) { + acceptance.GetEnvOrSkipTest(t, "ARM_CLIENT_ID") + acceptance.WorkspaceLevel(t, acceptance.Step{ + Template: groupsDataSourceTemplate, + Check: checkGroupsDataSourcePopulated(t), + }) +} + +func TestAccGroupsDataSourceOnAWS(t *testing.T) { + acceptance.GetEnvOrSkipTest(t, "TEST_EC2_INSTANCE_PROFILE") + acceptance.WorkspaceLevel(t, acceptance.Step{ + Template: groupsDataSourceTemplate, + Check: checkGroupsDataSourcePopulated(t), + }) +} + +func TestAccGroupsDataSourceOnGCP(t *testing.T) { + acceptance.GetEnvOrSkipTest(t, "GOOGLE_CREDENTIALS") + acceptance.WorkspaceLevel(t, acceptance.Step{ + Template: groupsDataSourceTemplate, + Check: checkGroupsDataSourcePopulated(t), + }) +} diff --git a/scim/data_groups_test.go b/scim/data_groups_test.go new file mode 100644 index 0000000000..c7dbe53da2 --- /dev/null +++ b/scim/data_groups_test.go @@ -0,0 +1,351 @@ +package scim + +import ( + "testing" + + "github.com/databricks/terraform-provider-databricks/qa" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDataSourceGroupsEmpty(t *testing.T) { + qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "GET", + Resource: `/api/2.0/preview/scim/v2/Groups?filter=displayName%20co%20%22nonexistent%22`, + Response: GroupList{ + Resources: []Group{}, + }, + }, + }, + Resource: DataSourceGroups(), + HCL: `filter = "displayName co \"nonexistent\""`, + Read: true, + NonWritable: true, + ID: "_", + }.ApplyAndExpectData(t, map[string]any{ + "groups": []any{}, + }) +} + + +func TestDataSourceGroupsMultiple(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "GET", + Resource: `/api/2.0/preview/scim/v2/Groups?filter=displayName%20co%20%22team%22`, + Response: GroupList{ + Resources: []Group{ + { + DisplayName: "team-alpha", + ID: "g1", + ExternalID: "ext1", + Members: []ComplexValue{ + { + Ref: "Users/u1", + Value: "user1", + }, + { + Ref: "ServicePrincipals/sp1", + Value: "service-principal-1", + }, + { + Ref: "Groups/g3", + Value: "child-group-1", + }, + }, + Groups: []ComplexValue{ + { + Value: "parent-group-1", + }, + }, + Roles: []ComplexValue{ + { + Value: "instance-profile-1", + }, + }, + }, + { + DisplayName: "team-beta", + ID: "g2", + ExternalID: "ext2", + Members: []ComplexValue{ + { + Ref: "Users/u2", + Value: "user2", + }, + { + Ref: "Users/u3", + Value: "user3", + }, + }, + }, + }, + }, + }, + }, + Resource: DataSourceGroups(), + HCL: `filter = "displayName co \"team\""`, + Read: true, + NonWritable: true, + ID: "_", + }.Apply(t) + require.NoError(t, err) + + groups := d.Get("groups").([]any) + require.Len(t, groups, 2) + + // Groups should be sorted alphabetically by display name + group1 := groups[0].(map[string]any) + group2 := groups[1].(map[string]any) + + // Verify team-alpha group + assert.Equal(t, "team-alpha", group1["display_name"]) + assert.Equal(t, "ext1", group1["external_id"]) + assert.Equal(t, "groups/team-alpha", group1["acl_principal_id"]) + + // Verify members are properly categorized in team-alpha + assertContains(t, group1["users"], "user1") + assertContains(t, group1["service_principals"], "service-principal-1") + assertContains(t, group1["child_groups"], "child-group-1") + assertContains(t, group1["groups"], "parent-group-1") + assertContains(t, group1["instance_profiles"], "instance-profile-1") + + // Verify team-beta group + assert.Equal(t, "team-beta", group2["display_name"]) + assert.Equal(t, "ext2", group2["external_id"]) + assert.Equal(t, "groups/team-beta", group2["acl_principal_id"]) + + // Verify members are properly categorized in team-beta + assertContains(t, group2["users"], "user2") + assertContains(t, group2["users"], "user3") +} + +func TestDataSourceGroupsSingleGroup(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "GET", + Resource: `/api/2.0/preview/scim/v2/Groups?filter=displayName%20eq%20%22admins%22`, + Response: GroupList{ + Resources: []Group{ + { + DisplayName: "admins", + ID: "admin-group", + ExternalID: "external-admin", + Members: []ComplexValue{ + { + Ref: "Users/admin1", + Value: "admin-user-1", + }, + { + Ref: "ServicePrincipals/sp-admin", + Value: "admin-service-principal", + }, + }, + Roles: []ComplexValue{ + { + Value: "admin-instance-profile", + }, + { + Value: "backup-instance-profile", + }, + }, + }, + }, + }, + }, + }, + Resource: DataSourceGroups(), + HCL: `filter = "displayName eq \"admins\""`, + Read: true, + NonWritable: true, + ID: "_", + }.Apply(t) + require.NoError(t, err) + + groups := d.Get("groups").([]any) + require.Len(t, groups, 1) + + group := groups[0].(map[string]any) + assert.Equal(t, "admins", group["display_name"]) + assert.Equal(t, "external-admin", group["external_id"]) + assert.Equal(t, "groups/admins", group["acl_principal_id"]) + + // Verify members are properly categorized + assertContains(t, group["users"], "admin-user-1") + assertContains(t, group["service_principals"], "admin-service-principal") + assertContains(t, group["instance_profiles"], "admin-instance-profile") + assertContains(t, group["instance_profiles"], "backup-instance-profile") +} + +func TestDataSourceGroupsError(t *testing.T) { + _, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "GET", + Resource: `/api/2.0/preview/scim/v2/Groups?filter=displayName%20co%20%22test%22`, + Status: 500, + Response: map[string]any{ + "error": "Internal Server Error", + }, + }, + }, + Resource: DataSourceGroups(), + HCL: `filter = "displayName co \"test\""`, + Read: true, + NonWritable: true, + ID: "_", + }.Apply(t) + require.Error(t, err) +} + +func TestDataSourceGroupsComplexMembers(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "GET", + Resource: `/api/2.0/preview/scim/v2/Groups?filter=displayName%20co%20%22complex%22`, + Response: GroupList{ + Resources: []Group{ + { + DisplayName: "complex-group", + ID: "complex-id", + Members: []ComplexValue{ + { + Ref: "Users/u1", + Value: "zebra-user", + }, + { + Ref: "Users/u2", + Value: "alpha-user", + }, + { + Ref: "ServicePrincipals/sp1", + Value: "zebra-sp", + }, + { + Ref: "ServicePrincipals/sp2", + Value: "alpha-sp", + }, + { + Ref: "Groups/g1", + Value: "zebra-group", + }, + { + Ref: "Groups/g2", + Value: "alpha-group", + }, + }, + Groups: []ComplexValue{ + { + Value: "zebra-parent", + }, + { + Value: "alpha-parent", + }, + }, + Roles: []ComplexValue{ + { + Value: "zebra-role", + }, + { + Value: "alpha-role", + }, + }, + }, + }, + }, + }, + }, + Resource: DataSourceGroups(), + HCL: `filter = "displayName co \"complex\""`, + Read: true, + NonWritable: true, + ID: "_", + }.Apply(t) + require.NoError(t, err) + + groups := d.Get("groups").([]any) + require.Len(t, groups, 1) + + group := groups[0].(map[string]any) + assert.Equal(t, "complex-group", group["display_name"]) + assert.Equal(t, "groups/complex-group", group["acl_principal_id"]) + + // Verify all users are properly categorized and sorted + assertContains(t, group["users"], "alpha-user") + assertContains(t, group["users"], "zebra-user") + + // Verify all service principals are properly categorized and sorted + assertContains(t, group["service_principals"], "alpha-sp") + assertContains(t, group["service_principals"], "zebra-sp") + + // Verify all child groups are properly categorized and sorted + assertContains(t, group["child_groups"], "alpha-group") + assertContains(t, group["child_groups"], "zebra-group") + + // Verify all parent groups are properly categorized and sorted + assertContains(t, group["groups"], "alpha-parent") + assertContains(t, group["groups"], "zebra-parent") + + // Verify all instance profiles are properly categorized and sorted + assertContains(t, group["instance_profiles"], "alpha-role") + assertContains(t, group["instance_profiles"], "zebra-role") + + // Verify that all members are included in the members set + assertContains(t, group["members"], "alpha-user") + assertContains(t, group["members"], "zebra-user") + assertContains(t, group["members"], "alpha-sp") + assertContains(t, group["members"], "zebra-sp") + assertContains(t, group["members"], "alpha-group") + assertContains(t, group["members"], "zebra-group") +} + +// Test that groups are sorted by display name +func TestDataSourceGroupsSorting(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "GET", + Resource: `/api/2.0/preview/scim/v2/Groups?filter=displayName%20co%20%22sort%22`, + Response: GroupList{ + Resources: []Group{ + { + DisplayName: "sort-zebra", + ID: "z-id", + }, + { + DisplayName: "sort-alpha", + ID: "a-id", + }, + { + DisplayName: "sort-beta", + ID: "b-id", + }, + }, + }, + }, + }, + Resource: DataSourceGroups(), + HCL: `filter = "displayName co \"sort\""`, + Read: true, + NonWritable: true, + ID: "_", + }.Apply(t) + require.NoError(t, err) + + groups := d.Get("groups").([]any) + require.Len(t, groups, 3) + + // Verify sorting by display name + group1 := groups[0].(map[string]any) + group2 := groups[1].(map[string]any) + group3 := groups[2].(map[string]any) + + assert.Equal(t, "sort-alpha", group1["display_name"]) + assert.Equal(t, "sort-beta", group2["display_name"]) + assert.Equal(t, "sort-zebra", group3["display_name"]) +}