Use .bazelci/build_bazel_binaries.yml for downstream pipelines (#1526)

In downstream pipelines, we need to build and upload Bazel binaries at
HEAD before testing downstream projects with it. Previously, we were
generating the "Build Bazel" steps for each supported platform from the
postsubmit.yml config. This was broken by

bazelbuild/bazel@3f93e7b,
because now there are multiple jobs for the same platform. See
https://buildkite.com/bazel/bazel-at-head-plus-downstream/builds/2815

This PR switches all downstream pipelines to use
build_bazel_binaries.yml config, which has the minimal steps for each
platform and builds both bazel and bazel_nojdk binaries.

Author: meteorcloudy

buildkite/terraform/bazel-testing/main.tf

@@ -175,15 +175,15 @@ resource "buildkite_pipeline" "bazelisk-plus-incompatible-flags" {
 resource "buildkite_pipeline" "bazel-at-head-plus-disabled" {
   name = "Bazel@HEAD + Disabled"
   repository = "https://github.com/bazelbuild/bazel.git"
-  steps = templatefile("pipeline.yml.tpl", { envs = {}, steps = { commands = ["curl -sS \"https://raw.githubusercontent.com/bazelbuild/continuous-integration/testing/buildkite/bazelci.py?$(date +%s)\" -o bazelci.py", "python3.6 bazelci.py bazel_downstream_pipeline --http_config=https://raw.githubusercontent.com/bazelbuild/bazel/master/.bazelci/presubmit.yml --test_disabled_projects | tee /dev/tty | buildkite-agent pipeline upload"] } })
+  steps = templatefile("pipeline.yml.tpl", { envs = {}, steps = { commands = ["curl -sS \"https://raw.githubusercontent.com/bazelbuild/continuous-integration/testing/buildkite/bazelci.py?$(date +%s)\" -o bazelci.py", "python3.6 bazelci.py bazel_downstream_pipeline --file_config=.bazelci/build_bazel_binaries.yml --test_disabled_projects | tee /dev/tty | buildkite-agent pipeline upload"] } })
   description = "Test disabled downstream projects to see if they are already fixed."
   default_branch = "master"
 }
 
 resource "buildkite_pipeline" "bazel-at-head-plus-downstream" {
   name = "Bazel@HEAD + Downstream"
   repository = "https://github.com/bazelbuild/bazel.git"
-  steps = templatefile("pipeline.yml.tpl", { envs = {}, steps = { commands = ["curl -sS \"https://raw.githubusercontent.com/bazelbuild/continuous-integration/testing/buildkite/bazelci.py?$(date +%s)\" -o bazelci.py", "python3.6 bazelci.py bazel_downstream_pipeline --http_config=https://raw.githubusercontent.com/bazelbuild/bazel/master/.bazelci/presubmit.yml | tee /dev/tty | buildkite-agent pipeline upload"] } })
+  steps = templatefile("pipeline.yml.tpl", { envs = {}, steps = { commands = ["curl -sS \"https://raw.githubusercontent.com/bazelbuild/continuous-integration/testing/buildkite/bazelci.py?$(date +%s)\" -o bazelci.py", "python3.6 bazelci.py bazel_downstream_pipeline --file_config=.bazelci/build_bazel_binaries.yml | tee /dev/tty | buildkite-agent pipeline upload"] } })
   description = "Test Bazel@HEAD + downstream projects@last_green_commit"
   default_branch = "master"
 }

buildkite/terraform/bazel/main.tf

@@ -1095,7 +1095,7 @@ resource "buildkite_pipeline" "rules-swift-swift" {
 resource "buildkite_pipeline" "bazel-at-head-plus-disabled" {
   name = "Bazel@HEAD + Disabled"
   repository = "https://github.com/bazelbuild/bazel.git"
-  steps = templatefile("pipeline.yml.tpl", { envs = {}, steps = { commands = ["curl -sS \"https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/bazelci.py?$(date +%s)\" -o bazelci.py", "python3.6 bazelci.py bazel_downstream_pipeline --http_config=https://raw.githubusercontent.com/bazelbuild/bazel/master/.bazelci/presubmit.yml --test_disabled_projects | tee /dev/tty | buildkite-agent pipeline upload"] } })
+  steps = templatefile("pipeline.yml.tpl", { envs = {}, steps = { commands = ["curl -sS \"https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/bazelci.py?$(date +%s)\" -o bazelci.py", "python3.6 bazelci.py bazel_downstream_pipeline --file_config=.bazelci/build_bazel_binaries.yml --test_disabled_projects | tee /dev/tty | buildkite-agent pipeline upload"] } })
   description = "Test disabled downstream projects to see if they are already fixed."
   default_branch = "master"
   team = [{ access_level = "MANAGE_BUILD_AND_READ", slug = "bazel-sheriffs" }, { access_level = "BUILD_AND_READ", slug = "downstream-pipeline-triggerers" }, { access_level = "READ_ONLY", slug = "bazel" }]
@@ -1673,7 +1673,7 @@ resource "buildkite_pipeline" "tensorflow" {
 resource "buildkite_pipeline" "bazel-at-head-plus-downstream" {
   name = "Bazel@HEAD + Downstream"
   repository = "https://github.com/bazelbuild/bazel.git"
-  steps = templatefile("pipeline.yml.tpl", { envs = {}, steps = { commands = ["curl -sS \"https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/bazelci.py?$(date +%s)\" -o bazelci.py", "python3.6 bazelci.py bazel_downstream_pipeline --file_config=.bazelci/postsubmit.yml | tee /dev/tty | buildkite-agent pipeline upload"] } })
+  steps = templatefile("pipeline.yml.tpl", { envs = {}, steps = { commands = ["curl -sS \"https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/bazelci.py?$(date +%s)\" -o bazelci.py", "python3.6 bazelci.py bazel_downstream_pipeline --file_config=.bazelci/build_bazel_binaries.yml | tee /dev/tty | buildkite-agent pipeline upload"] } })
   description = "Test Bazel@HEAD + downstream projects@last_green_commit"
   default_branch = "master"
   team = [{ access_level = "MANAGE_BUILD_AND_READ", slug = "bazel-sheriffs" }, { access_level = "BUILD_AND_READ", slug = "downstream-pipeline-triggerers" }, { access_level = "BUILD_AND_READ", slug = "bazel" }]