Merge main

.ci/containers/downstream-builder/generate_downstream.sh

@@ -4,7 +4,7 @@ set -e
 
 function clone_repo() {
     SCRATCH_OWNER=modular-magician
-    UPSTREAM_BRANCH=main
+    UPSTREAM_BRANCH=$BASE_BRANCH
     if [ "$REPO" == "terraform" ]; then
         if [ "$VERSION" == "ga" ]; then
             UPSTREAM_OWNER=hashicorp
@@ -25,11 +25,12 @@ function clone_repo() {
         LOCAL_PATH=$GOPATH/src/github.com/GoogleCloudPlatform/terraform-google-conversion
     elif [ "$REPO" == "terraform-google-conversion" ]; then
         UPSTREAM_OWNER=GoogleCloudPlatform
-        UPSTREAM_BRANCH=main
         GH_REPO=terraform-google-conversion
         LOCAL_PATH=$GOPATH/src/github.com/GoogleCloudPlatform/terraform-google-conversion
     elif [ "$REPO" == "tf-oics" ]; then
-        UPSTREAM_BRANCH=master
+        if [ "$UPSTREAM_BRANCH" == "main" ]; then
+            UPSTREAM_BRANCH=master
+        fi
         UPSTREAM_OWNER=terraform-google-modules
         GH_REPO=docs-examples
         LOCAL_PATH=$GOPATH/src/github.com/terraform-google-modules/docs-examples
@@ -45,7 +46,9 @@ function clone_repo() {
     GITHUB_PATH=https://modular-magician:$GITHUB_TOKEN@github.com/$UPSTREAM_OWNER/$GH_REPO
     SCRATCH_PATH=https://modular-magician:$GITHUB_TOKEN@github.com/$SCRATCH_OWNER/$GH_REPO
     mkdir -p "$(dirname $LOCAL_PATH)"
-    git clone $GITHUB_PATH $LOCAL_PATH
+
+    echo "BASE_BRANCH: $BASE_BRANCH"
+    git clone $GITHUB_PATH $LOCAL_PATH --branch $UPSTREAM_BRANCH
 }
 
 if [ $# -lt 4 ]; then
@@ -67,6 +70,12 @@ mkdir ../mm-$REPO-$VERSION-$COMMAND
 cp -rp ./. ../mm-$REPO-$VERSION-$COMMAND
 pushd ../mm-$REPO-$VERSION-$COMMAND
 
+# for backwards-compatibility
+if [ -z "$BASE_BRANCH" ]; then
+    BASE_BRANCH=main
+fi
+
+
 clone_repo
 
 git config --local user.name "Modular Magician"
@@ -112,20 +121,13 @@ if [ "$REPO" == "terraform-google-conversion" ]; then
 
     pushd $LOCAL_PATH
 
-    if [ "$VERSION" == "ga" ]; then
-      if [ "$COMMAND" == "downstream" ]; then
-        go get -d github.com/hashicorp/terraform-provider-google@main
-      else
-        go mod edit -replace github.com/hashicorp/terraform-provider-google=github.com/$SCRATCH_OWNER/terraform-provider-google@$BRANCH
-      fi
-    elif [ "$VERSION" == "beta" ]; then
-      if [ "$COMMAND" == "downstream" ]; then
-        go get -d github.com/hashicorp/terraform-provider-google-beta@main
-      else
-        go mod edit -replace github.com/hashicorp/terraform-provider-google-beta=github.com/$SCRATCH_OWNER/terraform-provider-google-beta@$BRANCH
-      fi
+    if [ "$COMMAND" == "downstream" ]; then
+      go get -d github.com/hashicorp/terraform-provider-google-beta@$BASE_BRANCH
+    else
+      go mod edit -replace github.com/hashicorp/terraform-provider-google-beta=github.com/$SCRATCH_OWNER/terraform-provider-google-beta@$BRANCH
     fi
 
+
     go mod tidy
 
     # the following build can fail which results in a subsequent failure to push to tfv repository.
@@ -155,6 +157,11 @@ else
         fi
         pushd ../
         make tpgtools OUTPUT_PATH=$LOCAL_PATH VERSION=$VERSION
+
+        # Only generate TeamCity-related file for TPG and TPGB
+        if [ "$VERSION" == "ga" ] || [ "$VERSION" == "beta" ]; then
+            make teamcity-servicemap-generate OUTPUT_PATH=$LOCAL_PATH VERSION=$VERSION
+        fi
         popd
     fi
 fi
@@ -177,7 +184,7 @@ if [ "$REPO" == "terraform" ]; then
 fi
 
 PR_NUMBER=$(curl -L -s -H "Authorization: token ${GITHUB_TOKEN}" \
-    "https://api.github.com/repos/GoogleCloudPlatform/magic-modules/pulls?state=closed&base=main&sort=updated&direction=desc" | \
+    "https://api.github.com/repos/GoogleCloudPlatform/magic-modules/pulls?state=closed&base=$BASE_BRANCH&sort=updated&direction=desc" | \
     jq -r ".[] | if .merge_commit_sha == \"$REFERENCE\" then .number else empty end")
 if [ "$COMMITTED" == "true" ] && [ "$COMMAND" == "downstream" ] && [ "$CHANGELOG" == "true" ]; then
     # Add the changelog entry!
@@ -222,4 +229,4 @@ if [ "$COMMITTED" == "true" ] && [ "$COMMAND" == "downstream" ]; then
         "https://api.github.com/repos/$UPSTREAM_OWNER/$GH_REPO/pulls/$NEW_PR_NUMBER/merge"
 fi
 
-popd
+popd

.ci/containers/downstream-waiter/wait_for_commit.sh

@@ -6,10 +6,18 @@ if [ $# -lt 3 ]; then
     exit 1
 fi
 
-SYNC_BRANCH=$1
+SYNC_BRANCH_PREFIX=$1
 BASE_BRANCH=$2
 SHA=$3
 
+if [ "$BASE_BRANCH" == "main" ]; then
+    SYNC_BRANCH=$SYNC_BRANCH_PREFIX
+else
+    SYNC_BRANCH=$SYNC_BRANCH_PREFIX-$BASE_BRANCH
+fi
+
+echo "SYNC_BRANCH: $SYNC_BRANCH"
+
 if git merge-base --is-ancestor $SHA origin/$SYNC_BRANCH; then
     echo "Found $SHA in history of $SYNC_BRANCH - dying to avoid double-generating that commit."
     exit 1
@@ -25,4 +33,4 @@ while true; do
         git fetch origin $SYNC_BRANCH
     fi
     sleep 5
-done
+done

.ci/containers/gcb-terraform-vcr-tester/test_terraform_vcr.sh

@@ -54,11 +54,25 @@ function update_status {
 
 update_status "pending"
 
+# for backwards-compatibility
+if [ -z "$BASE_BRANCH" ]; then
+  BASE_BRANCH=main
+else
+  echo "BASE_BRANCH: $BASE_BRANCH"
+fi
+
 set +e
 # cassette retrieval
 mkdir fixtures
-gsutil -m -q cp gs://ci-vcr-cassettes/beta/fixtures/* fixtures/
-# copy branch specific cassettes over master. This might fail but that's ok if the folder doesnt exist
+if [ "$BASE_BRANCH" != "FEATURE-BRANCH-major-release-5.0.0" ]; then
+  # pull main cassettes (major release uses branch specific casssettes as primary ones)
+  gsutil -m -q cp gs://ci-vcr-cassettes/beta/fixtures/* fixtures/
+fi
+if [ "$BASE_BRANCH" != "main" ]; then
+  # copy feature branch specific cassettes over main. This might fail but that's ok if the folder doesnt exist
+  gsutil -m -q cp gs://ci-vcr-cassettes/beta/refs/branches/$BASE_BRANCH/fixtures/* fixtures/
+fi
+# copy PR branch specific cassettes over main. This might fail but that's ok if the folder doesnt exist
 gsutil -m -q cp gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-$pr_number/fixtures/* fixtures/
 
 echo $SA_KEY > sa_key.json

.ci/containers/membership-checker/REVIEWER_ASSIGNMENT_COMMENT.md

@@ -1,21 +1,3 @@
-Hello!  I am a robot who works on Magic Modules PRs.
+Hello! I am a robot. It looks like you are a community contributor. @{{reviewer}}, a repository maintainer, has been assigned to review your changes. If you have not received review feedback within 2 business days, please leave a comment on this PR asking them to take a look.
 
-I've detected that you're a community contributor. @{{reviewer}}, a repository maintainer, has been assigned to assist you and help review your changes. 
-
-<details>
-  <summary>:question: First time contributing? Click here for more details</summary>
-
----
-
-Your assigned reviewer will help review your code by: 
-* Ensuring it's backwards compatible, covers common error cases, etc.
-* Summarizing the change into a user-facing changelog note.
-* Passes tests, either our "VCR" suite, a set of presubmit tests, or with manual test runs.
-
-You can help make sure that review is quick by running local tests and ensuring they're passing in between each push you make to your PR's branch. Also, try to leave a comment with each push you make, as pushes generally don't generate emails.
-
-If your reviewer doesn't get back to you within a week after your most recent change, please feel free to leave a comment on the issue asking them to take a look! In the absence of a dedicated review dashboard most maintainers manage their pending reviews through email, and those will sometimes get lost in their inbox.
-
----
-
-</details>
+You can help make sure that review is quick by [doing a self-review](https://googlecloudplatform.github.io/magic-modules/contribute/review-pr/) and by [running impacted tests locally](https://googlecloudplatform.github.io/magic-modules/get-started/run-provider-tests/).

.ci/containers/vcr-cassette-merger/vcr_merge.sh

@@ -4,25 +4,40 @@ set -e
 
 REFERENCE=$1
 
+# for backwards-compatibility
+if [ -z "$BASE_BRANCH" ]; then
+  BASE_BRANCH=main
+else
+  echo "BASE_BRANCH: $BASE_BRANCH"
+fi
+
 PR_NUMBER=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" \
-    "https://api.github.com/repos/GoogleCloudPlatform/magic-modules/pulls?state=closed&base=main&sort=updated&direction=desc" | \
+    "https://api.github.com/repos/GoogleCloudPlatform/magic-modules/pulls?state=closed&base=$BASE_BRANCH&sort=updated&direction=desc" | \
     jq -r ".[] | if .merge_commit_sha == \"$REFERENCE\" then .number else empty end")
 
 set +e
 gsutil ls gs://ci-vcr-cassettes/refs/heads/auto-pr-$PR_NUMBER/fixtures/
 if [ $? -eq 0 ]; then
   # We have recorded new cassettes for this branch
-  gsutil -m cp gs://ci-vcr-cassettes/refs/heads/auto-pr-$PR_NUMBER/fixtures/* gs://ci-vcr-cassettes/fixtures/
+    if [ "$BASE_BRANCH" == "main" ]; then
+      gsutil -m cp gs://ci-vcr-cassettes/refs/heads/auto-pr-$PR_NUMBER/fixtures/* gs://ci-vcr-cassettes/fixtures/
+    else
+      gsutil -m cp gs://ci-vcr-cassettes/refs/heads/auto-pr-$PR_NUMBER/fixtures/* gs://ci-vcr-cassettes/refs/branches/$BASE_BRANCH/fixtures/
+    fi
   gsutil -m rm -r gs://ci-vcr-cassettes/refs/heads/auto-pr-$PR_NUMBER/
 fi
 
 # Beta cassettes
 gsutil ls gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-$PR_NUMBER/fixtures/
 if [ $? -eq 0 ]; then
   # We have recorded new cassettes for this branch
-  gsutil -m cp gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-$PR_NUMBER/fixtures/* gs://ci-vcr-cassettes/beta/fixtures/
+    if [ "$BASE_BRANCH" == "main" ]; then
+      gsutil -m cp gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-$PR_NUMBER/fixtures/* gs://ci-vcr-cassettes/beta/fixtures/
+    else
+      gsutil -m cp gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-$PR_NUMBER/fixtures/* gs://ci-vcr-cassettes/beta/refs/branches/$BASE_BRANCH/fixtures/
+    fi
   gsutil -m rm -r gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-$PR_NUMBER/
 fi
 
 
-set -e
+set -e

.ci/gcb-generate-diffs-new.yml

@@ -76,6 +76,8 @@ steps:
       secretEnv: ["GITHUB_TOKEN"]
       id: tpg-head
       waitFor: ["merged"]
+      env:
+        - BASE_BRANCH=$_BASE_BRANCH
       args:
           - 'head'
           - 'terraform'
@@ -86,6 +88,8 @@ steps:
       id: tpg-base
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["merged"]
+      env:
+        - BASE_BRANCH=$_BASE_BRANCH
       args:
           - 'base'
           - 'terraform'
@@ -96,6 +100,8 @@ steps:
       secretEnv: ["GITHUB_TOKEN"]
       id: tpgb-head
       waitFor: ["merged"]
+      env:
+        - BASE_BRANCH=$_BASE_BRANCH
       args:
           - 'head'
           - 'terraform'
@@ -106,6 +112,8 @@ steps:
       id: tpgb-base
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["merged"]
+      env:
+        - BASE_BRANCH=$_BASE_BRANCH
       args:
           - 'base'
           - 'terraform'
@@ -116,25 +124,31 @@ steps:
       id: tgc-head
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["merged", "tpg-head"]
+      env:
+        - BASE_BRANCH=$_BASE_BRANCH
       args:
           - 'head'
           - 'terraform-google-conversion'
-          - 'ga'
+          - 'beta'
           - $_PR_NUMBER
 
     - name: 'gcr.io/graphite-docker-images/downstream-builder'
       id: tgc-base
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["merged", "tpg-base"]
+      env:
+        - BASE_BRANCH=$_BASE_BRANCH
       args:
           - 'base'
           - 'terraform-google-conversion'
-          - 'ga'
+          - 'beta'
           - $_PR_NUMBER
 
     - name: 'gcr.io/graphite-docker-images/downstream-builder'
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["merged"]
+      env:
+        - BASE_BRANCH=$_BASE_BRANCH
       args:
           - 'head'
           - 'tf-oics'
@@ -144,6 +158,8 @@ steps:
     - name: 'gcr.io/graphite-docker-images/downstream-builder'
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["merged"]
+      env:
+        - BASE_BRANCH=$_BASE_BRANCH
       args:
           - 'base'
           - 'tf-oics'
@@ -256,6 +272,8 @@ steps:
       id: gcb-tpg-vcr-test
       secretEnv: ["GITHUB_TOKEN", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"]
       waitFor: ["diff"]
+      env:
+        - BASE_BRANCH=$_BASE_BRANCH
       args:
           - $_PR_NUMBER
           - $COMMIT_SHA