diff --git a/Makefile b/Makefile index 55b9257..02f7b3e 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,7 @@ docker_generate_docs: -e ENABLE_BPMETADATA='1' \ -v "$(CURDIR)":/workspace \ $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ - /bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs' + /bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs display' # Alias for backwards compatibility .PHONY: generate_docs diff --git a/examples/bigquery-separate-pub-sub/README.md b/examples/bigquery-separate-pub-sub/README.md new file mode 100644 index 0000000..0e0d5c3 --- /dev/null +++ b/examples/bigquery-separate-pub-sub/README.md @@ -0,0 +1,35 @@ +# Bigquery Example + +This example illustrates how to use the `pub` amd `sub` module. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | The project ID to manage the Pub/Sub resources | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| project\_id | The project ID | +| topic\_labels | The labels of the Pub/Sub topic created | +| topic\_name | The name of the Pub/Sub topic created | + + + +## Requirements + +The following sections describe the requirements which must be met in +order to invoke this example. The requirements of the +[root module][root-module-requirements] must be met. + +## Usage + +To provision this example, populate `terraform.tfvars` with the [required variables](#inputs) and run the following commands within +this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/bigquery-separate-pub-sub/main.tf b/examples/bigquery-separate-pub-sub/main.tf new file mode 100644 index 0000000..66dd1d3 --- /dev/null +++ b/examples/bigquery-separate-pub-sub/main.tf @@ -0,0 +1,75 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "google" { + region = "europe-west1" +} + +module "pub" { + source = "terraform-google-modules/pubsub/google//modules/pub" + version = "~> 7.0" + + project_id = var.project_id + topic = "cft-tf-pub-topic-bigquery" + topic_labels = { + foo_label = "foo_value" + bar_label = "bar_value" + } +} + +module "sub" { + source = "terraform-google-modules/pubsub/google//modules/sub" + version = "~> 7.0" + + project_id = var.project_id + topic = "cft-tf-pub-topic-bigquery" + + bigquery_subscriptions = [ + { + name = "sub_example_subscription" + table = "${var.project_id}:sub_example_dataset.sub_example_table" + }, + ] + + depends_on = [ + google_bigquery_table.test + ] + +} + +resource "google_bigquery_dataset" "test" { + project = var.project_id + dataset_id = "sub_example_dataset" + location = "europe-west1" +} + +resource "google_bigquery_table" "test" { + project = var.project_id + deletion_protection = false + table_id = "sub_example_table" + dataset_id = google_bigquery_dataset.test.dataset_id + + schema = < +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | The project ID to manage the Pub/Sub resources | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| bucket\_name | The name of the Cloud Storage bucket created | +| project\_id | The project ID | +| topic\_name | The name of the Pub/Sub topic created | + + + +## Requirements + +The following sections describe the requirements which must be met in +order to invoke this example. The requirements of the +[root module][root-module-requirements] must be met. + +## Usage + +To provision this example, populate `terraform.tfvars` with the [required variables](#inputs) and run the following commands within +this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/cloud_storage-separate-pub-sub/main.tf b/examples/cloud_storage-separate-pub-sub/main.tf new file mode 100644 index 0000000..35f8bbe --- /dev/null +++ b/examples/cloud_storage-separate-pub-sub/main.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2018-2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_id" "bucket_suffix" { + byte_length = 4 +} + +provider "google" { + region = "europe-west1" +} + +module "pub" { + source = "terraform-google-modules/pubsub/google//modules/pub" + version = "~> 7.0" + + project_id = var.project_id + topic = "cft-tf-pub-topic-cloud-storage" + + topic_labels = { + foo_label = "foo_value" + } +} + +module "sub" { + source = "terraform-google-modules/pubsub/google//modules/sub" + version = "~> 7.0" + + project_id = var.project_id + topic = "cft-tf-pub-topic-cloud-storage" + + cloud_storage_subscriptions = [ + { + name = "sub_example_bucket_subscription" + bucket = google_storage_bucket.test.name + + filename_prefix = "example_prefix_" + filename_suffix = "_example_suffix" + filename_datetime_format = "YYYY-MM-DD/hh_mm_ssZ" + ack_deadline_seconds = 300 + }, + ] +} + +resource "google_storage_bucket" "test" { + project = var.project_id + name = join("-", ["sub_test_bucket", random_id.bucket_suffix.hex]) + location = "europe-west1" + uniform_bucket_level_access = true +} diff --git a/examples/cloud_storage-separate-pub-sub/outputs.tf b/examples/cloud_storage-separate-pub-sub/outputs.tf new file mode 100644 index 0000000..3d329de --- /dev/null +++ b/examples/cloud_storage-separate-pub-sub/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2018-2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = var.project_id + description = "The project ID" +} + +output "bucket_name" { + value = google_storage_bucket.test.name + description = "The name of the Cloud Storage bucket created" +} + +output "topic_name" { + value = module.pub.topic + description = "The name of the Pub/Sub topic created" +} diff --git a/examples/cloud_storage-separate-pub-sub/variables.tf b/examples/cloud_storage-separate-pub-sub/variables.tf new file mode 100644 index 0000000..5abd8d8 --- /dev/null +++ b/examples/cloud_storage-separate-pub-sub/variables.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "The project ID to manage the Pub/Sub resources" +} diff --git a/examples/simple-separate-pub-sub/README.md b/examples/simple-separate-pub-sub/README.md new file mode 100644 index 0000000..52e3366 --- /dev/null +++ b/examples/simple-separate-pub-sub/README.md @@ -0,0 +1,35 @@ +# Simple Example + +This example illustrates how to use the `pub` and `sub` module. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | The project ID to manage the Pub/Sub resources | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| project\_id | The project ID | +| topic\_labels | The labels of the Pub/Sub topic created | +| topic\_name | The name of the Pub/Sub topic created | + + + +## Requirements + +The following sections describe the requirements which must be met in +order to invoke this example. The requirements of the +[root module][root-module-requirements] must be met. + +## Usage + +To provision this example, populate `terraform.tfvars` with the [required variables](#inputs) and run the following commands within +this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple-separate-pub-sub/main.tf b/examples/simple-separate-pub-sub/main.tf new file mode 100644 index 0000000..c873203 --- /dev/null +++ b/examples/simple-separate-pub-sub/main.tf @@ -0,0 +1,75 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "google" { + region = "us-central1" +} + +module "pub" { + source = "terraform-google-modules/pubsub/google//modules/pub" + version = "~> 7.0" + + project_id = var.project_id + topic = "cft-tf-pub-topic" + topic_labels = { + foo_label = "foo_value" + bar_label = "bar_value" + } + + schema = { + name = "pub-example" + type = "AVRO" + encoding = "JSON" + definition = "{\n \"type\" : \"record\",\n \"name\" : \"Avro\",\n \"fields\" : [\n {\n \"name\" : \"StringField\",\n \"type\" : \"string\"\n },\n {\n \"name\" : \"IntField\",\n \"type\" : \"int\"\n }\n ]\n}\n" + } +} + +module "sub" { + source = "terraform-google-modules/pubsub/google//modules/sub" + version = "~> 7.0" + + project_id = var.project_id + topic = module.pub.topic + + pull_subscriptions = [ + { + name = "sub-pull" + ack_deadline_seconds = 10 + enable_exactly_once_delivery = true + }, + { + name = "sub-pull2" + minimum_backoff = "10s" + maximum_backoff = "600s" + expiration_policy = "1209600s" // two weeks + } + ] + + push_subscriptions = [ + { + name = "sub-push" + push_endpoint = "https://${var.project_id}.appspot.com/" + x-goog-version = "v1beta1" + ack_deadline_seconds = 20 + expiration_policy = "1209600s" // two weeks + }, + { + name = "sub-push-default-expiration" + push_endpoint = "https://${var.project_id}.appspot.com/" + x-goog-version = "v1beta1" + }, + ] +} diff --git a/examples/simple-separate-pub-sub/outputs.tf b/examples/simple-separate-pub-sub/outputs.tf new file mode 100644 index 0000000..0d72909 --- /dev/null +++ b/examples/simple-separate-pub-sub/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = var.project_id + description = "The project ID" +} + +output "topic_name" { + value = module.pub.topic + description = "The name of the Pub/Sub topic created" +} + +output "topic_labels" { + value = module.pub.topic_labels + description = "The labels of the Pub/Sub topic created" +} diff --git a/examples/simple-separate-pub-sub/variables.tf b/examples/simple-separate-pub-sub/variables.tf new file mode 100644 index 0000000..5abd8d8 --- /dev/null +++ b/examples/simple-separate-pub-sub/variables.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "The project ID to manage the Pub/Sub resources" +} diff --git a/metadata.yaml b/metadata.yaml index 73edf13..8bf4987 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -30,15 +30,26 @@ spec: version: ">= 1.3" description: {} content: + subBlueprints: + - name: pub + location: modules/pub + - name: sub + location: modules/sub examples: - name: bigquery location: examples/bigquery + - name: bigquery-separate-pub-sub + location: examples/bigquery-separate-pub-sub - name: cloud_storage location: examples/cloud_storage + - name: cloud_storage-separate-pub-sub + location: examples/cloud_storage-separate-pub-sub - name: kms location: examples/kms - name: simple location: examples/simple + - name: simple-separate-pub-sub + location: examples/simple-separate-pub-sub - name: subscriptions_only location: examples/subscriptions_only interfaces: diff --git a/modules/pub/README.md b/modules/pub/README.md new file mode 100644 index 0000000..0eff07d --- /dev/null +++ b/modules/pub/README.md @@ -0,0 +1,93 @@ +# terraform-google-pubsub + +This module makes it easy to create Google Cloud Pub/Sub topic. + +## Compatibility +This module is meant for use with Terraform 0.13+ and tested using Terraform 1.0+. If you find incompatibilities using Terraform >=0.13, please open an issue. + If you haven't +[upgraded](https://www.terraform.io/upgrade-guides/0-13.html) and need a Terraform +0.12.x-compatible version of this module, the last released version +intended for Terraform 0.12.x is [v1.9.0](https://registry.terraform.io/modules/terraform-google-modules/-pubsub/google/v1.9.0). + +## Usage + +This is a simple usage of the module. Please see also a simple setup provided in the example directory. + +```hcl +module "pub" { + source = "terraform-google-modules/pubsub/google//modules/pub" + version = "~> 7.0" + + project_id = var.project_id + topic = "cft-tf-pub-topic" + topic_labels = { + foo_label = "foo_value" + bar_label = "bar_value" + } + + schema = { + name = "pub-example" + type = "AVRO" + encoding = "JSON" + definition = "{\n \"type\" : \"record\",\n \"name\" : \"Avro\",\n \"fields\" : [\n {\n \"name\" : \"StringField\",\n \"type\" : \"string\"\n },\n {\n \"name\" : \"IntField\",\n \"type\" : \"int\"\n }\n ]\n}\n" + } +} +``` + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| message\_storage\_policy | A map of storage policies. Default - inherit from organization's Resource Location Restriction policy. | `map(any)` | `{}` | no | +| project\_id | The project ID to manage the Pub/Sub resources. | `string` | n/a | yes | +| schema | Schema for the topic. |
object({
name = string
type = string
definition = string
encoding = string
})
| `null` | no | +| topic | The Pub/Sub topic name. | `string` | n/a | yes | +| topic\_kms\_key\_name | The resource name of the Cloud KMS CryptoKey to be used to protect access to messages published on this topic. | `string` | `null` | no | +| topic\_labels | A map of labels to assign to the Pub/Sub topic. | `map(string)` | `{}` | no | +| topic\_message\_retention\_duration | The minimum duration in seconds to retain a message after it is published to the topic. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| id | The ID of the Pub/Sub topic | +| topic | The name of the Pub/Sub topic | +| topic\_labels | Labels assigned to the Pub/Sub topic | +| uri | The URI of the Pub/Sub topic | + + + +## Requirements + +### Installation Dependencies + +- [Terraform](https://www.terraform.io/downloads.html) >= 0.13.0 +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) plugin >= v2.13 + +### Configure a Service Account + +In order to execute this module you must have a Service Account with the following: + +#### Roles + +- `roles/pubsub.admin` + +### Enable APIs + +In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: + +- Cloud Pub/Sub API + +#### Service Account Credentials + +You can pass the service account credentials into this module by setting the following environment variables: + +* `GOOGLE_CREDENTIALS` +* `GOOGLE_CLOUD_KEYFILE_JSON` +* `GCLOUD_KEYFILE_JSON` + +See more [details](https://www.terraform.io/docs/providers/google/provider_reference.html#configuration-reference). + +[v0.2.0]: https://registry.terraform.io/modules/terraform-google-modules/pubsub/google/0.2.0 +[terraform-0.12-upgrade]: https://www.terraform.io/upgrade-guides/0-12.html diff --git a/modules/pub/main.tf b/modules/pub/main.tf new file mode 100644 index 0000000..5f565cb --- /dev/null +++ b/modules/pub/main.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_pubsub_schema" "schema" { + count = var.schema != null ? 1 : 0 + project = var.project_id + name = var.schema.name + type = var.schema.type + definition = var.schema.definition +} + +resource "google_pubsub_topic" "topic" { + project = var.project_id + name = var.topic + labels = var.topic_labels + kms_key_name = var.topic_kms_key_name + message_retention_duration = var.topic_message_retention_duration + + dynamic "message_storage_policy" { + for_each = var.message_storage_policy + content { + allowed_persistence_regions = message_storage_policy.key == "allowed_persistence_regions" ? message_storage_policy.value : null + } + } + + dynamic "schema_settings" { + for_each = var.schema != null ? [var.schema] : [] + content { + schema = google_pubsub_schema.schema[0].id + encoding = lookup(schema_settings.value, "encoding", null) + } + } + depends_on = [google_pubsub_schema.schema] +} diff --git a/modules/pub/metadata.display.yaml b/modules/pub/metadata.display.yaml new file mode 100644 index 0000000..9277a82 --- /dev/null +++ b/modules/pub/metadata.display.yaml @@ -0,0 +1,54 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: blueprints.cloud.google.com/v1alpha1 +kind: BlueprintMetadata +metadata: + name: terraform-google-pubsub-pub-display + annotations: + config.kubernetes.io/local-config: "true" +spec: + info: + title: terraform-google-pubsub + source: + repo: https://github.com/terraform-google-modules/terraform-google-pubsub + sourceType: git + dir: /modules/pub + ui: + input: + variables: + create_topic: + name: create_topic + title: Create Topic + message_storage_policy: + name: message_storage_policy + title: Message Storage Policy + project_id: + name: project_id + title: Project Id + schema: + name: schema + title: Schema + topic: + name: topic + title: Topic + topic_kms_key_name: + name: topic_kms_key_name + title: Topic Kms Key Name + topic_labels: + name: topic_labels + title: Topic Labels + topic_message_retention_duration: + name: topic_message_retention_duration + title: Topic Message Retention Duration diff --git a/modules/pub/metadata.yaml b/modules/pub/metadata.yaml new file mode 100644 index 0000000..12786c9 --- /dev/null +++ b/modules/pub/metadata.yaml @@ -0,0 +1,109 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: blueprints.cloud.google.com/v1alpha1 +kind: BlueprintMetadata +metadata: + name: terraform-google-pubsub-pub + annotations: + config.kubernetes.io/local-config: "true" +spec: + info: + title: terraform-google-pubsub + source: + repo: https://github.com/terraform-google-modules/terraform-google-pubsub + sourceType: git + dir: /modules/pub + version: 8.0.1 + actuationTool: + flavor: Terraform + version: ">= 1.3" + description: {} + content: + examples: + - name: bigquery + location: examples/bigquery + - name: bigquery-separate-pub-sub + location: examples/bigquery-separate-pub-sub + - name: cloud_storage + location: examples/cloud_storage + - name: cloud_storage-separate-pub-sub + location: examples/cloud_storage-separate-pub-sub + - name: kms + location: examples/kms + - name: simple + location: examples/simple + - name: simple-separate-pub-sub + location: examples/simple-separate-pub-sub + - name: subscriptions_only + location: examples/subscriptions_only + interfaces: + variables: + - name: project_id + description: The project ID to manage the Pub/Sub resources. + varType: string + required: true + - name: topic + description: The Pub/Sub topic name. + varType: string + required: true + - name: topic_labels + description: A map of labels to assign to the Pub/Sub topic. + varType: map(string) + defaultValue: {} + - name: topic_message_retention_duration + description: The minimum duration in seconds to retain a message after it is published to the topic. + varType: string + - name: message_storage_policy + description: A map of storage policies. Default - inherit from organization's Resource Location Restriction policy. + varType: map(any) + defaultValue: {} + - name: topic_kms_key_name + description: The resource name of the Cloud KMS CryptoKey to be used to protect access to messages published on this topic. + varType: string + - name: schema + description: Schema for the topic. + varType: |- + object({ + name = string + type = string + definition = string + encoding = string + }) + outputs: + - name: id + description: The ID of the Pub/Sub topic + - name: topic + description: The name of the Pub/Sub topic + - name: topic_labels + description: Labels assigned to the Pub/Sub topic + - name: uri + description: The URI of the Pub/Sub topic + requirements: + roles: + - level: Project + roles: + - roles/pubsub.admin + - roles/resourcemanager.projectIamAdmin + - roles/bigquery.admin + - roles/storage.admin + services: + - cloudresourcemanager.googleapis.com + - pubsub.googleapis.com + - serviceusage.googleapis.com + - bigquery.googleapis.com + - storage.googleapis.com + providerVersions: + - source: hashicorp/google + version: ">= 6.2, < 7" diff --git a/modules/pub/outputs.tf b/modules/pub/outputs.tf new file mode 100644 index 0000000..d786bd9 --- /dev/null +++ b/modules/pub/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "topic" { + value = google_pubsub_topic.topic.name + description = "The name of the Pub/Sub topic" +} + +output "topic_labels" { + value = google_pubsub_topic.topic.labels + description = "Labels assigned to the Pub/Sub topic" +} + +output "id" { + value = google_pubsub_topic.topic.id + description = "The ID of the Pub/Sub topic" +} + +output "uri" { + value = "pubsub.googleapis.com/${google_pubsub_topic.topic.id}" + description = "The URI of the Pub/Sub topic" +} diff --git a/modules/pub/variables.tf b/modules/pub/variables.tf new file mode 100644 index 0000000..b9d50da --- /dev/null +++ b/modules/pub/variables.tf @@ -0,0 +1,60 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "The project ID to manage the Pub/Sub resources." +} + +variable "topic" { + type = string + description = "The Pub/Sub topic name." +} + +variable "topic_labels" { + type = map(string) + description = "A map of labels to assign to the Pub/Sub topic." + default = {} +} + +variable "topic_message_retention_duration" { + type = string + description = "The minimum duration in seconds to retain a message after it is published to the topic." + default = null +} + +variable "message_storage_policy" { + type = map(any) + description = "A map of storage policies. Default - inherit from organization's Resource Location Restriction policy." + default = {} +} + +variable "topic_kms_key_name" { + type = string + description = "The resource name of the Cloud KMS CryptoKey to be used to protect access to messages published on this topic." + default = null +} + +variable "schema" { + type = object({ + name = string + type = string + definition = string + encoding = string + }) + description = "Schema for the topic." + default = null +} diff --git a/modules/pub/versions.tf b/modules/pub/versions.tf new file mode 100644 index 0000000..8a95ad4 --- /dev/null +++ b/modules/pub/versions.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.3" + required_providers { + + google = { + source = "hashicorp/google" + version = ">= 6.2, < 7" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/terraform-google-pubsub:pub/v8.0.1" + } + +} diff --git a/modules/sub/README.md b/modules/sub/README.md new file mode 100644 index 0000000..da84257 --- /dev/null +++ b/modules/sub/README.md @@ -0,0 +1,111 @@ +# terraform-google-pubsub + +This module makes it easy to create Google Cloud Pub/Sub subscriptions associated with a given topic. + +## Compatibility +This module is meant for use with Terraform 0.13+ and tested using Terraform 1.0+. If you find incompatibilities using Terraform >=0.13, please open an issue. + If you haven't +[upgraded](https://www.terraform.io/upgrade-guides/0-13.html) and need a Terraform +0.12.x-compatible version of this module, the last released version +intended for Terraform 0.12.x is [v1.9.0](https://registry.terraform.io/modules/terraform-google-modules/-pubsub/google/v1.9.0). + +## Usage + +This is a simple usage of the module. Please see also a simple setup provided in the example directory. + +```hcl +module "sub" { + source = "terraform-google-modules/pubsub/google//modules/sub" + version = "~> 7.0" + + project_id = var.project_id + topic = module.pub.topic + + pull_subscriptions = [ + { + name = "sub-pull" + ack_deadline_seconds = 10 + enable_exactly_once_delivery = true + }, + { + name = "sub-pull2" + minimum_backoff = "10s" + maximum_backoff = "600s" + expiration_policy = "1209600s" // two weeks + } + ] + + push_subscriptions = [ + { + name = "sub-push" + push_endpoint = "https://${var.project_id}.appspot.com/" + x-goog-version = "v1beta1" + ack_deadline_seconds = 20 + expiration_policy = "1209600s" // two weeks + }, + { + name = "sub-push-default-expiration" + push_endpoint = "https://${var.project_id}.appspot.com/" + x-goog-version = "v1beta1" + }, + ] +} +``` + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| bigquery\_subscriptions | The list of the Bigquery push subscriptions. |
list(object({
name = string,
table = string,
use_topic_schema = optional(bool),
use_table_schema = optional(bool),
write_metadata = optional(bool),
drop_unknown_fields = optional(bool),
ack_deadline_seconds = optional(number),
retain_acked_messages = optional(bool),
message_retention_duration = optional(string),
enable_message_ordering = optional(bool),
expiration_policy = optional(string),
filter = optional(string),
dead_letter_topic = optional(string),
maximum_backoff = optional(string),
minimum_backoff = optional(string)
}))
| `[]` | no | +| cloud\_storage\_subscriptions | The list of the Cloud Storage push subscriptions. |
list(object({
name = string,
bucket = string,
filename_prefix = optional(string),
filename_suffix = optional(string),
filename_datetime_format = optional(string),
max_duration = optional(string),
max_bytes = optional(string),
max_messages = optional(string),
output_format = optional(string),
write_metadata = optional(bool),
use_topic_schema = optional(bool),
ack_deadline_seconds = optional(number),
retain_acked_messages = optional(bool),
message_retention_duration = optional(string),
enable_message_ordering = optional(bool),
expiration_policy = optional(string),
filter = optional(string),
dead_letter_topic = optional(string),
maximum_backoff = optional(string),
minimum_backoff = optional(string)
}))
| `[]` | no | +| grant\_bigquery\_project\_roles | Specify true if you want to add bigquery.metadataViewer and bigquery.dataEditor roles to the default Pub/Sub SA. | `bool` | `true` | no | +| grant\_token\_creator | Specify true if you want to add token creator role to the default Pub/Sub SA. | `bool` | `true` | no | +| project\_id | The project ID to manage the Pub/Sub resources. | `string` | n/a | yes | +| pull\_subscriptions | The list of the pull subscriptions. |
list(object({
name = string,
ack_deadline_seconds = optional(number),
expiration_policy = optional(string),
dead_letter_topic = optional(string),
max_delivery_attempts = optional(number),
retain_acked_messages = optional(bool),
message_retention_duration = optional(string),
maximum_backoff = optional(string),
minimum_backoff = optional(string),
filter = optional(string),
enable_message_ordering = optional(bool),
service_account = optional(string),
enable_exactly_once_delivery = optional(bool),
}))
| `[]` | no | +| push\_subscriptions | The list of the push subscriptions. |
list(object({
name = string,
ack_deadline_seconds = optional(number),
push_endpoint = optional(string),
x-goog-version = optional(string),
oidc_service_account_email = optional(string),
audience = optional(string),
expiration_policy = optional(string),
dead_letter_topic = optional(string),
retain_acked_messages = optional(bool),
message_retention_duration = optional(string),
max_delivery_attempts = optional(number),
maximum_backoff = optional(string),
minimum_backoff = optional(string),
filter = optional(string),
enable_message_ordering = optional(bool),
}))
| `[]` | no | +| subscription\_labels | A map of labels to assign to every Pub/Sub subscription. | `map(string)` | `{}` | no | +| topic | The Pub/Sub topic name. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| subscription\_names | The name list of Pub/Sub subscriptions | +| subscription\_paths | The path list of Pub/Sub subscriptions | + + + +## Requirements + +### Installation Dependencies + +- [Terraform](https://www.terraform.io/downloads.html) >= 0.13.0 +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) plugin >= v2.13 + +### Configure a Service Account + +In order to execute this module you must have a Service Account with the following: + +#### Roles + +- `roles/pubsub.admin` + +### Enable APIs + +In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: + +- Cloud Pub/Sub API + +#### Service Account Credentials + +You can pass the service account credentials into this module by setting the following environment variables: + +* `GOOGLE_CREDENTIALS` +* `GOOGLE_CLOUD_KEYFILE_JSON` +* `GCLOUD_KEYFILE_JSON` + +See more [details](https://www.terraform.io/docs/providers/google/provider_reference.html#configuration-reference). + +[v0.2.0]: https://registry.terraform.io/modules/terraform-google-modules/pubsub/google/0.2.0 +[terraform-0.12-upgrade]: https://www.terraform.io/upgrade-guides/0-12.html diff --git a/modules/sub/main.tf b/modules/sub/main.tf new file mode 100644 index 0000000..f3ff94b --- /dev/null +++ b/modules/sub/main.tf @@ -0,0 +1,359 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_project" "project" { + project_id = var.project_id +} + +locals { + default_ack_deadline_seconds = 10 + pubsub_svc_account_email = "service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" +} + +resource "google_project_iam_member" "bigquery_metadata_viewer_binding" { + count = var.grant_bigquery_project_roles && (length(var.bigquery_subscriptions) != 0) ? 1 : 0 + project = var.project_id + role = "roles/bigquery.metadataViewer" + member = "serviceAccount:${local.pubsub_svc_account_email}" +} + +resource "google_project_iam_member" "bigquery_data_editor_binding" { + count = var.grant_bigquery_project_roles && (length(var.bigquery_subscriptions) != 0) ? 1 : 0 + project = var.project_id + role = "roles/bigquery.dataEditor" + member = "serviceAccount:${local.pubsub_svc_account_email}" +} + +resource "google_project_iam_member" "storage_admin_binding" { + count = length(var.cloud_storage_subscriptions) != 0 ? 1 : 0 + project = var.project_id + role = "roles/storage.admin" + member = "serviceAccount:${local.pubsub_svc_account_email}" +} + +resource "google_project_iam_member" "token_creator_binding" { + count = var.grant_token_creator ? 1 : 0 + project = var.project_id + role = "roles/iam.serviceAccountTokenCreator" + member = "serviceAccount:${local.pubsub_svc_account_email}" + depends_on = [ + google_pubsub_subscription.push_subscriptions, + ] +} + +resource "google_pubsub_topic_iam_member" "push_topic_binding" { + for_each = { for i in var.push_subscriptions : i.name => i if i.dead_letter_topic != null } + + project = var.project_id + topic = each.value.dead_letter_topic + role = "roles/pubsub.publisher" + member = "serviceAccount:${local.pubsub_svc_account_email}" +} + +resource "google_pubsub_topic_iam_member" "pull_topic_binding" { + for_each = { for i in var.pull_subscriptions : i.name => i if i.dead_letter_topic != null } + + project = var.project_id + topic = each.value.dead_letter_topic + role = "roles/pubsub.publisher" + member = "serviceAccount:${local.pubsub_svc_account_email}" +} + +resource "google_pubsub_topic_iam_member" "bigquery_topic_binding" { + for_each = { for i in var.bigquery_subscriptions : i.name => i if i.dead_letter_topic != null } + + project = var.project_id + topic = each.value.dead_letter_topic + role = "roles/pubsub.publisher" + member = "serviceAccount:${local.pubsub_svc_account_email}" +} + +resource "google_pubsub_subscription_iam_member" "pull_subscription_binding" { + for_each = { for i in var.pull_subscriptions : i.name => i if i.dead_letter_topic != null } + + project = var.project_id + subscription = each.value.name + role = "roles/pubsub.subscriber" + member = "serviceAccount:${local.pubsub_svc_account_email}" + depends_on = [ + google_pubsub_subscription.pull_subscriptions, + ] + + lifecycle { + replace_triggered_by = [google_pubsub_subscription.pull_subscriptions[each.key]] + } +} + +resource "google_pubsub_subscription_iam_member" "push_subscription_binding" { + for_each = { for i in var.push_subscriptions : i.name => i if i.dead_letter_topic != null } + + project = var.project_id + subscription = each.value.name + role = "roles/pubsub.subscriber" + member = "serviceAccount:${local.pubsub_svc_account_email}" + depends_on = [ + google_pubsub_subscription.push_subscriptions, + ] + + lifecycle { + replace_triggered_by = [google_pubsub_subscription.push_subscriptions[each.key]] + } +} + +resource "google_pubsub_subscription_iam_member" "bigquery_subscription_binding" { + for_each = { for i in var.bigquery_subscriptions : i.name => i if i.dead_letter_topic != null } + + project = var.project_id + subscription = each.value.name + role = "roles/pubsub.subscriber" + member = "serviceAccount:${local.pubsub_svc_account_email}" + depends_on = [ + google_pubsub_subscription.bigquery_subscriptions, + ] +} + +resource "google_pubsub_subscription" "push_subscriptions" { + for_each = { for i in var.push_subscriptions : i.name => i } + + name = each.value.name + topic = var.topic + project = var.project_id + labels = var.subscription_labels + ack_deadline_seconds = each.value.ack_deadline_seconds != null ? each.value.ack_deadline_seconds : local.default_ack_deadline_seconds + message_retention_duration = each.value.message_retention_duration + retain_acked_messages = each.value.retain_acked_messages + filter = each.value.filter + enable_message_ordering = each.value.enable_message_ordering + dynamic "expiration_policy" { + // check if the 'expiration_policy' key exists, if yes, return a list containing it. + for_each = each.value.expiration_policy != null ? [each.value.expiration_policy] : [] + content { + ttl = expiration_policy.value + } + } + + dynamic "dead_letter_policy" { + for_each = each.value.dead_letter_topic != null ? [each.value.dead_letter_topic] : [] + content { + dead_letter_topic = each.value.dead_letter_topic != null ? each.value.dead_letter_topic : "" + max_delivery_attempts = each.value.max_delivery_attempts != null ? each.value.max_delivery_attempts : "" + } + } + + dynamic "retry_policy" { + for_each = each.value.maximum_backoff != null ? [each.value.maximum_backoff] : [] + content { + maximum_backoff = each.value.maximum_backoff != null ? each.value.maximum_backoff : "" + minimum_backoff = each.value.minimum_backoff != null ? each.value.minimum_backoff : "5" + } + } + + push_config { + push_endpoint = each.value["push_endpoint"] + + // FIXME: This should be programmable, but nested map isn't supported at this time. + // https://github.com/hashicorp/terraform/issues/2114 + attributes = { + x-goog-version = each.value.minimum_backoff != null ? "x-goog-version" : "v1" + } + + dynamic "oidc_token" { + for_each = each.value.oidc_service_account_email != null ? [true] : [] + content { + service_account_email = each.value.oidc_service_account_email != null ? each.value.oidc_service_account_email : "" + audience = each.value.audience != null ? each.value.audience : "" + } + } + } +} + +resource "google_pubsub_subscription" "pull_subscriptions" { + for_each = { for i in var.pull_subscriptions : i.name => i } + + name = each.value.name + topic = var.topic + project = var.project_id + labels = var.subscription_labels + enable_exactly_once_delivery = each.value.enable_exactly_once_delivery + ack_deadline_seconds = each.value.ack_deadline_seconds != null ? each.value.ack_deadline_seconds : local.default_ack_deadline_seconds + message_retention_duration = each.value.message_retention_duration + retain_acked_messages = each.value.retain_acked_messages + filter = each.value.filter + enable_message_ordering = each.value.enable_message_ordering + dynamic "expiration_policy" { + // check if the 'expiration_policy' key exists, if yes, return a list containing it. + for_each = each.value.expiration_policy != null ? [each.value.expiration_policy] : [] + content { + ttl = expiration_policy.value + } + } + + dynamic "dead_letter_policy" { + for_each = each.value.dead_letter_topic != null ? [each.value.dead_letter_topic] : [] + content { + dead_letter_topic = each.value.dead_letter_topic != null ? each.value.dead_letter_topic : "" + max_delivery_attempts = each.value.max_delivery_attempts != null ? each.value.max_delivery_attempts : "5" + } + } + + dynamic "retry_policy" { + for_each = each.value.maximum_backoff != null ? [each.value.maximum_backoff] : [] + content { + maximum_backoff = each.value.maximum_backoff != null ? each.value.maximum_backoff : "" + minimum_backoff = each.value.minimum_backoff != null ? each.value.minimum_backoff : "5" + } + } +} + +resource "google_pubsub_subscription" "bigquery_subscriptions" { + for_each = { for i in var.bigquery_subscriptions : i.name => i } + + name = each.value.name + topic = var.topic + project = var.project_id + labels = var.subscription_labels + ack_deadline_seconds = each.value.ack_deadline_seconds != null ? each.value.ack_deadline_seconds : local.default_ack_deadline_seconds + message_retention_duration = each.value.message_retention_duration + retain_acked_messages = each.value.retain_acked_messages + filter = each.value.filter + enable_message_ordering = each.value.enable_message_ordering + dynamic "expiration_policy" { + // check if the 'expiration_policy' key exists, if yes, return a list containing it. + for_each = each.value.expiration_policy != null ? [each.value.expiration_policy] : [] + content { + ttl = expiration_policy.value + } + } + + dynamic "dead_letter_policy" { + for_each = each.value.dead_letter_topic != null ? [each.value.dead_letter_topic] : [] + content { + dead_letter_topic = each.value.dead_letter_topic != null ? each.value.dead_letter_topic : "" + max_delivery_attempts = each.value.max_delivery_attempts != null ? each.value.max_delivery_attempts : "5" + } + } + + dynamic "retry_policy" { + for_each = each.value.maximum_backoff != null ? [each.value.maximum_backoff] : [] + content { + maximum_backoff = each.value.maximum_backoff != null ? each.value.maximum_backoff : "" + minimum_backoff = each.value.minimum_backoff != null ? each.value.minimum_backoff : "" + } + } + + bigquery_config { + table = each.value["table"] + use_topic_schema = each.value.use_topic_schema != null ? each.value.use_topic_schema : false + use_table_schema = each.value.use_table_schema != null ? each.value.use_table_schema : false + write_metadata = each.value.write_metadata != null ? each.value.write_metadata : false + drop_unknown_fields = each.value.drop_unknown_fields != null ? each.value.drop_unknown_fields : false + } + + depends_on = [ + google_project_iam_member.bigquery_metadata_viewer_binding, + google_project_iam_member.bigquery_data_editor_binding + ] +} + +resource "google_pubsub_subscription" "cloud_storage_subscriptions" { + for_each = { for i in var.cloud_storage_subscriptions : i.name => i } + + name = each.value.name + topic = var.topic + project = var.project_id + labels = var.subscription_labels + ack_deadline_seconds = each.value.ack_deadline_seconds != null ? each.value.ack_deadline_seconds : local.default_ack_deadline_seconds + message_retention_duration = each.value.message_retention_duration + retain_acked_messages = each.value.retain_acked_messages + filter = each.value.filter + enable_message_ordering = each.value.enable_message_ordering + dynamic "expiration_policy" { + // check if the 'expiration_policy' key exists, if yes, return a list containing it. + for_each = each.value.expiration_policy != null ? [each.value.expiration_policy] : [] + content { + ttl = expiration_policy.value + } + } + + dynamic "dead_letter_policy" { + for_each = each.value.dead_letter_topic != null ? [each.value.dead_letter_topic] : [] + content { + dead_letter_topic = each.value.dead_letter_topic != null ? each.value.dead_letter_topic : "" + max_delivery_attempts = each.value.max_delivery_attempts != null ? each.value.max_delivery_attempts : "5" + } + } + + dynamic "retry_policy" { + for_each = each.value.maximum_backoff != null ? [each.value.maximum_backoff] : [] + content { + maximum_backoff = each.value.maximum_backoff != null ? each.value.maximum_backoff : "" + minimum_backoff = each.value.minimum_backoff != null ? each.value.minimum_backoff : "" + } + } + + cloud_storage_config { + bucket = each.value["bucket"] + filename_prefix = each.value.filename_prefix + filename_suffix = each.value.filename_suffix + filename_datetime_format = each.value.filename_datetime_format + max_duration = each.value.max_duration + max_bytes = each.value.max_bytes + max_messages = each.value.max_messages + dynamic "avro_config" { + for_each = (each.value.output_format == "avro") ? [true] : [] + content { + write_metadata = each.value.write_metadata + use_topic_schema = each.value.use_topic_schema + } + } + } + + depends_on = [ + google_project_iam_member.storage_admin_binding + ] +} + +resource "google_pubsub_subscription_iam_member" "pull_subscription_sa_binding_subscriber" { + for_each = { for i in var.pull_subscriptions : i.name => i if i.service_account != null } + + project = var.project_id + subscription = each.value.name + role = "roles/pubsub.subscriber" + member = "serviceAccount:${each.value.service_account}" + depends_on = [ + google_pubsub_subscription.pull_subscriptions, + ] + + lifecycle { + replace_triggered_by = [google_pubsub_subscription.pull_subscriptions[each.key]] + } +} + +resource "google_pubsub_subscription_iam_member" "pull_subscription_sa_binding_viewer" { + for_each = { for i in var.pull_subscriptions : i.name => i if i.service_account != null } + + project = var.project_id + subscription = each.value.name + role = "roles/pubsub.viewer" + member = "serviceAccount:${each.value.service_account}" + depends_on = [ + google_pubsub_subscription.pull_subscriptions, + ] + + lifecycle { + replace_triggered_by = [google_pubsub_subscription.pull_subscriptions[each.key]] + } +} diff --git a/modules/sub/metadata.display.yaml b/modules/sub/metadata.display.yaml new file mode 100644 index 0000000..1a60733 --- /dev/null +++ b/modules/sub/metadata.display.yaml @@ -0,0 +1,60 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: blueprints.cloud.google.com/v1alpha1 +kind: BlueprintMetadata +metadata: + name: terraform-google-pubsub-sub-display + annotations: + config.kubernetes.io/local-config: "true" +spec: + info: + title: terraform-google-pubsub + source: + repo: https://github.com/terraform-google-modules/terraform-google-pubsub + sourceType: git + dir: /modules/sub + ui: + input: + variables: + bigquery_subscriptions: + name: bigquery_subscriptions + title: Bigquery Subscriptions + cloud_storage_subscriptions: + name: cloud_storage_subscriptions + title: Cloud Storage Subscriptions + create_subscriptions: + name: create_subscriptions + title: Create Subscriptions + grant_bigquery_project_roles: + name: grant_bigquery_project_roles + title: Grant Bigquery Project Roles + grant_token_creator: + name: grant_token_creator + title: Grant Token Creator + project_id: + name: project_id + title: Project Id + pull_subscriptions: + name: pull_subscriptions + title: Pull Subscriptions + push_subscriptions: + name: push_subscriptions + title: Push Subscriptions + subscription_labels: + name: subscription_labels + title: Subscription Labels + topic: + name: topic + title: Topic diff --git a/modules/sub/metadata.yaml b/modules/sub/metadata.yaml new file mode 100644 index 0000000..443c879 --- /dev/null +++ b/modules/sub/metadata.yaml @@ -0,0 +1,181 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: blueprints.cloud.google.com/v1alpha1 +kind: BlueprintMetadata +metadata: + name: terraform-google-pubsub-sub + annotations: + config.kubernetes.io/local-config: "true" +spec: + info: + title: terraform-google-pubsub + source: + repo: https://github.com/terraform-google-modules/terraform-google-pubsub + sourceType: git + dir: /modules/sub + version: 8.0.1 + actuationTool: + flavor: Terraform + version: ">= 1.3" + description: {} + content: + examples: + - name: bigquery + location: examples/bigquery + - name: bigquery-separate-pub-sub + location: examples/bigquery-separate-pub-sub + - name: cloud_storage + location: examples/cloud_storage + - name: cloud_storage-separate-pub-sub + location: examples/cloud_storage-separate-pub-sub + - name: kms + location: examples/kms + - name: simple + location: examples/simple + - name: simple-separate-pub-sub + location: examples/simple-separate-pub-sub + - name: subscriptions_only + location: examples/subscriptions_only + interfaces: + variables: + - name: project_id + description: The project ID to manage the Pub/Sub resources. + varType: string + required: true + - name: topic + description: The Pub/Sub topic name. + varType: string + required: true + - name: push_subscriptions + description: The list of the push subscriptions. + varType: |- + list(object({ + name = string, + ack_deadline_seconds = optional(number), + push_endpoint = optional(string), + x-goog-version = optional(string), + oidc_service_account_email = optional(string), + audience = optional(string), + expiration_policy = optional(string), + dead_letter_topic = optional(string), + retain_acked_messages = optional(bool), + message_retention_duration = optional(string), + max_delivery_attempts = optional(number), + maximum_backoff = optional(string), + minimum_backoff = optional(string), + filter = optional(string), + enable_message_ordering = optional(bool), + })) + defaultValue: [] + - name: pull_subscriptions + description: The list of the pull subscriptions. + varType: |- + list(object({ + name = string, + ack_deadline_seconds = optional(number), + expiration_policy = optional(string), + dead_letter_topic = optional(string), + max_delivery_attempts = optional(number), + retain_acked_messages = optional(bool), + message_retention_duration = optional(string), + maximum_backoff = optional(string), + minimum_backoff = optional(string), + filter = optional(string), + enable_message_ordering = optional(bool), + service_account = optional(string), + enable_exactly_once_delivery = optional(bool), + })) + defaultValue: [] + - name: bigquery_subscriptions + description: The list of the Bigquery push subscriptions. + varType: |- + list(object({ + name = string, + table = string, + use_topic_schema = optional(bool), + use_table_schema = optional(bool), + write_metadata = optional(bool), + drop_unknown_fields = optional(bool), + ack_deadline_seconds = optional(number), + retain_acked_messages = optional(bool), + message_retention_duration = optional(string), + enable_message_ordering = optional(bool), + expiration_policy = optional(string), + filter = optional(string), + dead_letter_topic = optional(string), + maximum_backoff = optional(string), + minimum_backoff = optional(string) + })) + defaultValue: [] + - name: cloud_storage_subscriptions + description: The list of the Cloud Storage push subscriptions. + varType: |- + list(object({ + name = string, + bucket = string, + filename_prefix = optional(string), + filename_suffix = optional(string), + filename_datetime_format = optional(string), + max_duration = optional(string), + max_bytes = optional(string), + max_messages = optional(string), + output_format = optional(string), + write_metadata = optional(bool), + use_topic_schema = optional(bool), + ack_deadline_seconds = optional(number), + retain_acked_messages = optional(bool), + message_retention_duration = optional(string), + enable_message_ordering = optional(bool), + expiration_policy = optional(string), + filter = optional(string), + dead_letter_topic = optional(string), + maximum_backoff = optional(string), + minimum_backoff = optional(string) + })) + defaultValue: [] + - name: subscription_labels + description: A map of labels to assign to every Pub/Sub subscription. + varType: map(string) + defaultValue: {} + - name: grant_bigquery_project_roles + description: Specify true if you want to add bigquery.metadataViewer and bigquery.dataEditor roles to the default Pub/Sub SA. + varType: bool + defaultValue: true + - name: grant_token_creator + description: Specify true if you want to add token creator role to the default Pub/Sub SA. + varType: bool + defaultValue: true + outputs: + - name: subscription_names + description: The name list of Pub/Sub subscriptions + - name: subscription_paths + description: The path list of Pub/Sub subscriptions + requirements: + roles: + - level: Project + roles: + - roles/pubsub.admin + - roles/resourcemanager.projectIamAdmin + - roles/bigquery.admin + - roles/storage.admin + services: + - cloudresourcemanager.googleapis.com + - pubsub.googleapis.com + - serviceusage.googleapis.com + - bigquery.googleapis.com + - storage.googleapis.com + providerVersions: + - source: hashicorp/google + version: ">= 6.2, < 7" diff --git a/modules/sub/outputs.tf b/modules/sub/outputs.tf new file mode 100644 index 0000000..f214870 --- /dev/null +++ b/modules/sub/outputs.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "subscription_names" { + value = concat( + values({ for k, v in google_pubsub_subscription.push_subscriptions : k => v.name }), + values({ for k, v in google_pubsub_subscription.pull_subscriptions : k => v.name }), + values({ for k, v in google_pubsub_subscription.bigquery_subscriptions : k => v.name }), + values({ for k, v in google_pubsub_subscription.cloud_storage_subscriptions : k => v.name }), + ) + + description = "The name list of Pub/Sub subscriptions" +} + +output "subscription_paths" { + value = concat( + values({ for k, v in google_pubsub_subscription.push_subscriptions : k => v.id }), + values({ for k, v in google_pubsub_subscription.pull_subscriptions : k => v.id }), + values({ for k, v in google_pubsub_subscription.bigquery_subscriptions : k => v.name }), + values({ for k, v in google_pubsub_subscription.cloud_storage_subscriptions : k => v.name }), + ) + + description = "The path list of Pub/Sub subscriptions" +} + diff --git a/modules/sub/variables.tf b/modules/sub/variables.tf new file mode 100644 index 0000000..acdb4e6 --- /dev/null +++ b/modules/sub/variables.tf @@ -0,0 +1,134 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "The project ID to manage the Pub/Sub resources." +} + +variable "topic" { + type = string + description = "The Pub/Sub topic name." +} + +variable "push_subscriptions" { + type = list(object({ + name = string, + ack_deadline_seconds = optional(number), + push_endpoint = optional(string), + x-goog-version = optional(string), + oidc_service_account_email = optional(string), + audience = optional(string), + expiration_policy = optional(string), + dead_letter_topic = optional(string), + retain_acked_messages = optional(bool), + message_retention_duration = optional(string), + max_delivery_attempts = optional(number), + maximum_backoff = optional(string), + minimum_backoff = optional(string), + filter = optional(string), + enable_message_ordering = optional(bool), + })) + description = "The list of the push subscriptions." + default = [] +} + +variable "pull_subscriptions" { + type = list(object({ + name = string, + ack_deadline_seconds = optional(number), + expiration_policy = optional(string), + dead_letter_topic = optional(string), + max_delivery_attempts = optional(number), + retain_acked_messages = optional(bool), + message_retention_duration = optional(string), + maximum_backoff = optional(string), + minimum_backoff = optional(string), + filter = optional(string), + enable_message_ordering = optional(bool), + service_account = optional(string), + enable_exactly_once_delivery = optional(bool), + })) + description = "The list of the pull subscriptions." + default = [] +} + +variable "bigquery_subscriptions" { + type = list(object({ + name = string, + table = string, + use_topic_schema = optional(bool), + use_table_schema = optional(bool), + write_metadata = optional(bool), + drop_unknown_fields = optional(bool), + ack_deadline_seconds = optional(number), + retain_acked_messages = optional(bool), + message_retention_duration = optional(string), + enable_message_ordering = optional(bool), + expiration_policy = optional(string), + filter = optional(string), + dead_letter_topic = optional(string), + maximum_backoff = optional(string), + minimum_backoff = optional(string) + })) + description = "The list of the Bigquery push subscriptions." + default = [] +} + +variable "cloud_storage_subscriptions" { + type = list(object({ + name = string, + bucket = string, + filename_prefix = optional(string), + filename_suffix = optional(string), + filename_datetime_format = optional(string), + max_duration = optional(string), + max_bytes = optional(string), + max_messages = optional(string), + output_format = optional(string), + write_metadata = optional(bool), + use_topic_schema = optional(bool), + ack_deadline_seconds = optional(number), + retain_acked_messages = optional(bool), + message_retention_duration = optional(string), + enable_message_ordering = optional(bool), + expiration_policy = optional(string), + filter = optional(string), + dead_letter_topic = optional(string), + maximum_backoff = optional(string), + minimum_backoff = optional(string) + })) + description = "The list of the Cloud Storage push subscriptions." + default = [] +} + +variable "subscription_labels" { + type = map(string) + description = "A map of labels to assign to every Pub/Sub subscription." + default = {} +} + +variable "grant_bigquery_project_roles" { + type = bool + description = "Specify true if you want to add bigquery.metadataViewer and bigquery.dataEditor roles to the default Pub/Sub SA." + default = true +} + +variable "grant_token_creator" { + type = bool + description = "Specify true if you want to add token creator role to the default Pub/Sub SA." + default = true +} diff --git a/modules/sub/versions.tf b/modules/sub/versions.tf new file mode 100644 index 0000000..3c53574 --- /dev/null +++ b/modules/sub/versions.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.3" + required_providers { + + google = { + source = "hashicorp/google" + version = ">= 6.2, < 7" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/terraform-google-pubsub:sub/v8.0.1" + } + +} diff --git a/test/integration/bigquery-separate-pub-sub/bigquery_separate_pub_sub_test.go b/test/integration/bigquery-separate-pub-sub/bigquery_separate_pub_sub_test.go new file mode 100755 index 0000000..fe0ebbf --- /dev/null +++ b/test/integration/bigquery-separate-pub-sub/bigquery_separate_pub_sub_test.go @@ -0,0 +1,61 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery_separate_pub_sub + +import ( + "fmt" + "testing" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" +) + +func TestBigquerySeparatePubSub(t *testing.T) { + bpt := tft.NewTFBlueprintTest(t) + + bpt.DefineVerify(func(assert *assert.Assertions) { + bpt.DefaultVerify(assert) + + projectId := bpt.GetStringOutput("project_id") + + op := gcloud.Runf(t, "pubsub topics describe cft-tf-pub-topic-bigquery --project=%s", projectId) + assert.Equal(fmt.Sprintf("projects/%s/topics/cft-tf-pub-topic-bigquery", projectId), op.Get("name").String(), "has expected name") + assert.Equal("bar_value", op.Get("labels.bar_label").String(), "has expected labels") + assert.Equal("foo_value", op.Get("labels.foo_label").String(), "has expected labels") + + op = gcloud.Runf(t, "pubsub subscriptions describe sub_example_subscription --project=%s", projectId) + assert.Equal("", op.Get("RetryPolicy.minimumBackoff").String(), "has expected minimum_backoff") + assert.Equal(fmt.Sprintf("projects/%s/subscriptions/sub_example_subscription", projectId), op.Get("name").String(), "has expected name") + assert.Equal(fmt.Sprintf("projects/%s/topics/cft-tf-pub-topic-bigquery", projectId), op.Get("topic").String(), "has expected topic") + assert.Equal("10", op.Get("ackDeadlineSeconds").String(), "has expected ackDeadlineSeconds") + assert.Equal(false, op.Get("enableExactlyOnceDelivery").Bool(), "has expected enable_exactly_once_delivery") + assert.Equal(fmt.Sprintf("%s:sub_example_dataset.sub_example_table", projectId), op.Get("bigqueryConfig.table").String(), "has expected table") + assert.Equal("604800s", op.Get("messageRetentionDuration").String(), "has expected message_retention_duration") + assert.Equal("2678400s", op.Get("expirationPolicy.ttl").String(), "has expected expiration_policy") + assert.Equal("", op.Get("filter").String(), "has expected filter") + assert.Equal("", op.Get("deadLetterPolicy.deadLetterTopic").String(), "has expected dead_letter_topic") + assert.Equal("", op.Get("deadLetterPolicy.maxDeliveryAttempts").String(), "has expected dead_letter_topic") + assert.Equal("", op.Get("retryPolicy.maximumBackoff").String(), "has expected maximum_backoff") + assert.Equal("", op.Get("retryPolicy.minimumBackoff").String(), "has expected minimum_backoff") + assert.Equal(false, op.Get("bigqueryConfig.useTopicSchema").Bool(), "has expected use_topic_schema") + assert.Equal(false, op.Get("bigqueryConfig.useTableSchema").Bool(), "has expected use_table_schema") + assert.Equal(false, op.Get("bigqueryConfig.writeMetadata").Bool(), "has expected write_metadata") + assert.Equal(false, op.Get("bigqueryConfig.dropUnknownFields").Bool(), "has expected drop_unknown_fields") + assert.Equal(false, op.Get("bigqueryConfig.retainAckedMessages").Bool(), "has expected retain_acked") + }) + + bpt.Test() +} diff --git a/test/integration/cloud_storage-separate-pub-sub/cloud_storage_separate_pub_sub_test.go b/test/integration/cloud_storage-separate-pub-sub/cloud_storage_separate_pub_sub_test.go new file mode 100755 index 0000000..634b534 --- /dev/null +++ b/test/integration/cloud_storage-separate-pub-sub/cloud_storage_separate_pub_sub_test.go @@ -0,0 +1,59 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloud_storage_separate_pub_sub + +import ( + "fmt" + "testing" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" +) + +func TestCloudStorageSeparatePubSub(t *testing.T) { + bpt := tft.NewTFBlueprintTest(t) + + bpt.DefineVerify(func(assert *assert.Assertions) { + bpt.DefaultVerify(assert) + + projectId := bpt.GetStringOutput("project_id") + + op := gcloud.Runf(t, "pubsub topics describe cft-tf-pub-topic-cloud-storage --project=%s", projectId) + assert.Equal(fmt.Sprintf("projects/%s/topics/cft-tf-pub-topic-cloud-storage", projectId), op.Get("name").String(), "has expected name") + assert.Equal("foo_value", op.Get("labels.foo_label").String(), "has expected labels") + + op = gcloud.Runf(t, "pubsub subscriptions describe sub_example_bucket_subscription --project=%s", projectId) + assert.Equal("", op.Get("RetryPolicy.minimumBackoff").String(), "has expected minimum_backoff") + assert.Equal(fmt.Sprintf("projects/%s/subscriptions/sub_example_bucket_subscription", projectId), op.Get("name").String(), "has expected name") + assert.Equal(fmt.Sprintf("projects/%s/topics/cft-tf-pub-topic-cloud-storage", projectId), op.Get("topic").String(), "has expected topic") + assert.Equal("300", op.Get("ackDeadlineSeconds").String(), "has expected ackDeadlineSeconds") + assert.Equal(false, op.Get("enableExactlyOnceDelivery").Bool(), "has expected enable_exactly_once_delivery") + assert.Equal("604800s", op.Get("messageRetentionDuration").String(), "has expected message_retention_duration") + assert.Equal("2678400s", op.Get("expirationPolicy.ttl").String(), "has expected expiration_policy") + assert.Equal("", op.Get("filter").String(), "has expected filter") + assert.Equal("", op.Get("deadLetterPolicy.deadLetterTopic").String(), "has expected dead_letter_topic") + assert.Equal("", op.Get("deadLetterPolicy.maxDeliveryAttempts").String(), "has expected dead_letter_topic") + assert.Equal("", op.Get("retryPolicy.maximumBackoff").String(), "has expected maximum_backoff") + assert.Equal("", op.Get("retryPolicy.minimumBackoff").String(), "has expected minimum_backoff") + assert.Equal("example_prefix_", op.Get("cloudStorageConfig.filenamePrefix").String(), "has expected filename_prefix") + assert.Equal("_example_suffix", op.Get("cloudStorageConfig.filenameSuffix").String(), "has expected filename_suffix") + assert.Equal("300s", op.Get("cloudStorageConfig.maxDuration").String(), "has expected filename_prefix") + assert.Equal("YYYY-MM-DD/hh_mm_ssZ", op.Get("cloudStorageConfig.filenameDatetimeFormat").String(), "has expected filename_datetime_format") + assert.Equal(false, op.Get("cloudStorageConfig.avroConfig").Bool(), "has expected avro_config") + }) + + bpt.Test() +} diff --git a/test/integration/simple-separate-pub-sub/simple_separate_pub_sub_test.go b/test/integration/simple-separate-pub-sub/simple_separate_pub_sub_test.go new file mode 100755 index 0000000..6f8f566 --- /dev/null +++ b/test/integration/simple-separate-pub-sub/simple_separate_pub_sub_test.go @@ -0,0 +1,78 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package simple_separate_pub_sub + +import ( + "fmt" + "testing" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" +) + +func TestSimpleSeparatePubSub(t *testing.T) { + bpt := tft.NewTFBlueprintTest(t) + + bpt.DefineVerify(func(assert *assert.Assertions) { + bpt.DefaultVerify(assert) + + projectId := bpt.GetStringOutput("project_id") + + op := gcloud.Runf(t, "pubsub topics describe cft-tf-pub-topic --project=%s", projectId) + assert.Equal(fmt.Sprintf("projects/%s/topics/cft-tf-pub-topic", projectId), op.Get("name").String(), "has expected name") + assert.Equal("bar_value", op.Get("labels.bar_label").String(), "has expected labels") + assert.Equal("foo_value", op.Get("labels.foo_label").String(), "has expected labels") + + op = gcloud.Runf(t, "pubsub subscriptions describe sub-pull --project=%s", projectId) + assert.Equal(fmt.Sprintf("projects/%s/subscriptions/sub-pull", projectId), op.Get("name").String(), "has expected name") + assert.Equal(fmt.Sprintf("projects/%s/topics/cft-tf-pub-topic", projectId), op.Get("topic").String(), "has expected topic") + assert.Equal("10", op.Get("ackDeadlineSeconds").String(), "has expected ackDeadlineSeconds") + assert.Equal(true, op.Get("enableExactlyOnceDelivery").Bool(), "has expected enable_exactly_once_delivery") + assert.Equal("604800s", op.Get("messageRetentionDuration").String(), "has expected message_retention_duration") + assert.Equal("2678400s", op.Get("expirationPolicy.ttl").String(), "has expected expiration_policy") + assert.Equal("", op.Get("filter").String(), "has expected filter") + assert.Equal("", op.Get("deadLetterPolicy.deadLetterTopic").String(), "has expected dead_letter_topic") + assert.Equal("", op.Get("deadLetterPolicy.maxDeliveryAttempts").String(), "has expected dead_letter_topic") + assert.Equal("", op.Get("retryPolicy.maximumBackoff").String(), "has expected maximum_backoff") + assert.Equal("", op.Get("retryPolicy.minimumBackoff").String(), "has expected minimum_backoff") + op = gcloud.Runf(t, "pubsub subscriptions describe sub-push --project=%s", projectId) + assert.Equal(fmt.Sprintf("projects/%s/subscriptions/sub-push", projectId), op.Get("name").String(), "has expected name") + assert.Equal(fmt.Sprintf("projects/%s/topics/cft-tf-pub-topic", projectId), op.Get("topic").String(), "has expected topic") + assert.Equal("20", op.Get("ackDeadlineSeconds").String(), "has expected ackDeadlineSeconds") + assert.Equal(fmt.Sprintf("https://%s.appspot.com/", projectId), op.Get("pushConfig.pushEndpoint").String(), "has expected pushEndpoint") + assert.Equal("604800s", op.Get("messageRetentionDuration").String(), "has expected message_retention_duration") + assert.Equal("1209600s", op.Get("expirationPolicy.ttl").String(), "has expected expiration_policy") + assert.Equal("", op.Get("filter").String(), "has expected filter") + assert.Equal("", op.Get("deadLetterPolicy.deadLetterTopic").String(), "has expected dead_letter_topic") + assert.Equal("", op.Get("deadLetterPolicy.maxDeliveryAttempts").String(), "has expected dead_letter_topic") + assert.Equal("", op.Get("retryPolicy.maximumBackoff").String(), "has expected maximum_backoff") + assert.Equal("", op.Get("retryPolicy.minimumBackoff").String(), "has expected minimum_backoff") + assert.Equal(fmt.Sprintf("https://%s.appspot.com/", projectId), op.Get("pushConfig.pushEndpoint").String(), "has expected push_endpoint") + assert.Equal("", op.Get("pushConfig.oidcToken.serviceAccountEmail").String(), "has expected oidcToken.service_account_email") + assert.Equal("", op.Get("pushConfig.oidcToken.audience").String(), "has expected oidcToken.audience") + + op = gcloud.Runf(t, "pubsub subscriptions describe sub-pull2 --project=%s", projectId) + assert.Equal("600s", op.Get("retryPolicy.maximumBackoff").String(), "has expected maximum_backoff") + assert.Equal("10s", op.Get("retryPolicy.minimumBackoff").String(), "has expected minimum_backoff") + assert.Equal("1209600s", op.Get("expirationPolicy.ttl").String(), "has expected expiration_policy") + + op = gcloud.Runf(t, "pubsub schemas describe pub-example --project=%s", projectId) + assert.Equal(fmt.Sprintf("projects/%s/schemas/pub-example", projectId), op.Get("name").String(), "has expected name") + + }) + + bpt.Test() +}