fix pipeline permission bug

Author: violet2016

concourse/pipeline-release.yml

@@ -1,10 +1,69 @@
 groups:
 - name: All
   jobs:
-  - test_greenplumpython_light_centos7
-  - test_greenplumpython_light_ubuntu18
+  - test_plcontainer_centos7
+  - test_plcontainer_ubuntu18
   - greenplumPython_release
 
+ccp_create_params_anchor: &ccp_default_params
+  action: create
+  delete_on_failure: true
+  generate_random_name: true
+  terraform_source: ccp_src/google/
+
+ccp_vars_anchor: &ccp_default_vars
+  instance_type: n1-standard-4
+  region: {{google-region}}
+  zone: {{google-zone}}
+  disk_size: 100
+
+ccp_gen_cluster_default_params_anchor: &ccp_gen_cluster_default_params
+  AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}}
+  AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}}
+  AWS_DEFAULT_REGION: {{aws-region}}
+  BUCKET_PATH: clusters-google/
+  BUCKET_NAME: {{tf-bucket-name}}
+  CLOUD_PROVIDER: google
+
+ccp_destroy_anchor: &ccp_destroy
+  put: terraform
+  params:
+    action: destroy
+    env_name_file: terraform/name
+    terraform_source: ccp_src/google/
+    vars:
+      aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required.
+      aws_ebs_volume_type: standard
+  get_params:
+    action: destroy
+
+set_failed_anchor: &set_failed
+  do:
+  - task: on_failure_set_failed
+    config:
+      platform: linux
+      image_resource:
+        type: docker-image
+        source:
+          repository: pivotaldata/ccp
+          tag: "7"
+      inputs:
+        - name: ccp_src
+        - name: terraform
+      run:
+        path: 'ccp_src/google/ccp_failed_test.sh'
+      params:
+        GOOGLE_CREDENTIALS: {{google-service-account-key}}
+        GOOGLE_PROJECT_ID: {{google-project-id}}
+        GOOGLE_ZONE: {{google-zone}}
+        GOOGLE_SERVICE_ACCOUNT: {{google-service-account}}
+        AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}}
+        AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}}
+        AWS_DEFAULT_REGION: {{tf-machine-region}}
+        BUCKET_PATH: clusters-google/
+        BUCKET_NAME: {{tf-bucket-name}}
+#####################################################
+
 resource_types:
 - name: gcs
   type: docker-image
@@ -31,22 +90,46 @@ resources:
     repository: pivotaldata/gpdb6-ubuntu18.04-test
     tag: latest
 
-# Github Source Codes
-- name: gpdb_src
-  type: git
-  source:
-    branch: 6X_STABLE
-    uri: https://github.com/greenplum-db/gpdb.git
-    ignore_paths:
-    - gpdb-doc/*
-    - README*
 
 - name: GreenplumPython_src
   type: git
   source:
     branch: master
     uri: https://github.com/greenplum-db/GreenplumPython.git
 
+# ccp_src tag_filter is not maintained by gpdb_common-ci-secrets.yml
+# so we just pull the laster ccp_src without tag_filter
+- name: ccp_src
+  type: git
+  source:
+    branch: {{ccp-git-branch}}
+    private_key: {{ccp-git-key}}
+    uri: {{ccp-git-remote}}
+
+- name: plcontainer_src
+  type: git
+  source:
+    branch: master
+    uri: https://github.com/greenplum-db/plcontainer.git
+
+- name: terraform
+  type: terraform
+  source:
+    env:
+      AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}}
+      AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}}
+      GOOGLE_CREDENTIALS: {{google-service-account-key}}
+    vars:
+      project_id: {{google-project-id}}
+    storage:
+      access_key_id: {{tf-machine-access-key-id}}
+      secret_access_key: {{tf-machine-secret-access-key}}
+      region_name: {{aws-region}}
+      # This is not parameterized, on purpose. All tfstates will go to this spot,
+      # and different teams will place there clusters' tfstate files under different paths
+      bucket: {{tf-bucket-name}}
+      bucket_path: clusters-google/
+
 - name: bin_gpdb_centos7
   type: gcs
   source:
@@ -61,6 +144,34 @@ resources:
     json_key: ((concourse-gcs-resources-service-account-key))
     versioned_file: 6X_STABLE/bin_gpdb_ubuntu18.04/bin_gpdb.tar.gz
 
+- name: plcontainer_docker_image_centos_python
+  type: gcs
+  source:
+    bucket: {{gcs-bucket-intermediates}}
+    json_key: {{concourse-gcs-resources-service-account-key}}
+    versioned_file: plcontainer/published/gpdb6/plcontainer-python3-images-devel.tar.gz
+
+- name: bin_plcontainer_centos7
+  type: gcs
+  source:
+    bucket: {{gcs-bucket}}
+    json_key: {{concourse-gcs-resources-service-account-key}}
+    regexp: plcontainer/released/gpdb6/plcontainer-(2.*)-gp6-rhel7_x86_64.gppkg
+
+- name: bin_plcontainer_ubuntu18
+  type: gcs
+  source:
+    bucket: {{gcs-bucket}}
+    json_key: {{concourse-gcs-resources-service-account-key}}
+    regexp: plcontainer/released/gpdb6/plcontainer-(2.*)-gp6-ubuntu18.04_x86_64.gppkg
+
+- name: plcontainer_gpdb_ubuntu18_build_lib
+  type: gcs
+  source:
+    bucket: {{gcs-bucket-intermediates}}
+    json_key: {{concourse-gcs-resources-service-account-key}}
+    versioned_file: plcontainer/published/gpdb6/plcontainer-concourse-ubuntu18-lib.tar.gz
+
 - name: greenplumPython-release
   type: gcs
   source:
@@ -69,37 +180,90 @@ resources:
     regexp: pl-analysis-suite/released/gpdb6/greenplumPython-(.*).tar.gz
 
 jobs:
-- name: test_greenplumpython_light_centos7
+# test GrenplumPython for plcontainer
+- name: test_plcontainer_centos7
   plan:
   - aggregate:
-    - get: gpdb_src
+    - get: plcontainer_src
     - get: GreenplumPython_src
       trigger: true
-    - get: bin_gpdb
+    # this name is needed by ccp to gencluster
+    - get: gpdb_binary
       resource: bin_gpdb_centos7
-    - get: bin_plr
-      resource: bin_plr_centos7
+    - get: ccp_src
     - get: centos-gpdb-dev-7
-  - task: test_greenplumr_plr_centos7
-    file: GreenplumPython_src/concourse/tasks/test_plr.yml
-    image: centos-gpdb-dev-7
+    - get: plcontainer_pyclient_docker_image
+      resource: plcontainer_docker_image_centos_python
+      trigger: true
+    - get: bin_plcontainer
+      resource: bin_plcontainer_centos7
+      trigger: true
+    - get: plcontainer_gpdb_ubuntu18_build_lib
+  - put: terraform
     params:
+      <<: *ccp_default_params
+      vars:
+        <<: *ccp_default_vars
+        PLATFORM: centos7
+  - task: gen_cluster
+    file: ccp_src/ci/tasks/gen_cluster.yml
+    params:
+      <<: *ccp_gen_cluster_default_params
+      PLATFORM: centos7
+  - task: gpinitsystem
+    file: ccp_src/ci/tasks/gpinitsystem.yml
+  - task: test_greenplumpython_plcontainer
+    file: GreenplumPython_src/concourse/tasks/test_plcontainer.yml
+    params:
+      MODE: full
+      platform: centos7
+    image: centos-gpdb-dev-7
+    on_success:
+      <<: *ccp_destroy
+  ensure:
+    <<: *set_failed
 
-- name: test_greenplumpython_light_ubuntu18
+- name: test_plcontainer_ubuntu18
   plan:
   - aggregate:
-    - get: gpdb_src
+    - get: plcontainer_src
     - get: GreenplumPython_src
       trigger: true
-    - get: bin_gpdb
+    # this name is needed by ccp to gencluster
+    - get: gpdb_binary
       resource: bin_gpdb_ubuntu18
-    - get: bin_plr
-      resource: bin_plr_ubuntu18
+    - get: ccp_src
     - get: ubuntu18-image-test
-  - task: test_greenplumr_plr
-    file: GreenplumPython_src/concourse/tasks/test_plr.yml
-    image: ubuntu18-image-test
+    - get: plcontainer_pyclient_docker_image
+      resource: plcontainer_docker_image_centos_python
+      trigger: true
+    - get: bin_plcontainer
+      resource: bin_plcontainer_ubuntu18
+      trigger: true
+    - get: plcontainer_gpdb_ubuntu18_build_lib
+  - put: terraform
     params:
+      <<: *ccp_default_params
+      vars:
+        <<: *ccp_default_vars
+        PLATFORM: ubuntu18.04
+  - task: gen_cluster
+    file: ccp_src/ci/tasks/gen_cluster.yml
+    params:
+      <<: *ccp_gen_cluster_default_params
+      PLATFORM: ubuntu18.04
+  - task: gpinitsystem
+    file: ccp_src/ci/tasks/gpinitsystem.yml
+  - task: test_greenplumpython_plcontainer
+    file: GreenplumPython_src/concourse/tasks/test_plcontainer.yml
+    params:
+      MODE: full
+      platform: ubuntu18
+    image: ubuntu18-image-test
+    on_success:
+      <<: *ccp_destroy
+  ensure:
+    <<: *set_failed
 
 # Release GreenplumPython GP Package
 - name: greenplumPython_release
@@ -109,7 +273,7 @@ jobs:
     - get: centos-gpdb-dev-7
     - get: GreenplumPython_src
   - task: plcontainer_gpdb_build
-    file: GreenplumPython_src/concourse/tasks/greenplumPython_release.yml
+    file: GreenplumPython_src/concourse/tasks/greenplumpython_release.yml
     image: centos-gpdb-dev-7
   - aggregate:
     - put: greenplumPython-release

concourse/pipeline.yml

@@ -176,7 +176,7 @@ resources:
 
 jobs:
 
-# test GrenplumR for plcontainer
+# test GrenplumPython for plcontainer
 - name: test_plcontainer_centos7
   plan:
   - aggregate: