From a6761d31ed03ff3760f3f8f20fad174884518fe2 Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Mon, 3 May 2021 20:21:30 +0200 Subject: [PATCH] build: use platform from shared dev-infra package for remote execution Uses the new shared RBE platform from the dev-infra package. We introduced a shared Bazel platform for remote execution builds using Google cloud. Previously we used `bazel_toolchains` for providing the platform w/ additional CPP and Java toolchains `bazel_toolchains` no longer provides default toolchains with the latest version, but provides a tool (linux and windows only) for generating toolchain/platforms, which then need to be checked into the repository. This is quite inconvenient and cumbersome (especially with no macOS support), so we just provided our own platform and CPP toolchain within `@angular/dev-infra-private`. This is more simple than all the effort we'd need to make the toolchain generation tool work (while it would also increase the amount of checked-in sources significantly; with more unused toolchains for CPP or Java) See: angular/angular#41767. --- .bazelrc | 21 ++++++++------------- .bazelversion | 2 -- WORKSPACE | 35 ----------------------------------- package.json | 2 +- renovate.json | 2 +- tools/BUILD.bazel | 22 +++++----------------- yarn.lock | 21 ++++++++++++++++----- 7 files changed, 31 insertions(+), 74 deletions(-) diff --git a/.bazelrc b/.bazelrc index 872b401b8a..12d0607308 100644 --- a/.bazelrc +++ b/.bazelrc @@ -105,20 +105,15 @@ build:remote --remote_executor=remotebuildexecution.googleapis.com build:remote --remote_timeout=600 build:remote --jobs=150 - # Setup the toolchain and platform for the remote build execution. The platform -# is automatically configured by the "rbe_autoconfig" rule in the project workpsace. -build:remote --host_javabase=@rbe_ubuntu1604_angular//java:jdk -build:remote --javabase=@rbe_ubuntu1604_angular//java:jdk -build:remote --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 -build:remote --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 -build:remote --crosstool_top=@rbe_ubuntu1604_angular//cc:toolchain -build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 -build:remote --extra_toolchains=@rbe_ubuntu1604_angular//config:cc-toolchain -build:remote --extra_execution_platforms=//tools:rbe_ubuntu1604-angular -build:remote --host_platform=//tools:rbe_ubuntu1604-angular -build:remote --platforms=//tools:rbe_ubuntu1604-angular +# Setup the toolchain and platform for the remote build execution. The platform +# is provided by the shared dev-infra package and targets k8 remote containers. +build:remote --crosstool_top=@npm//@angular/dev-infra-private/bazel/remote-execution/cpp:cc_toolchain_suite +build:remote --extra_toolchains=@npm//@angular/dev-infra-private/bazel/remote-execution/cpp:cc_toolchain +build:remote --extra_execution_platforms=//tools:rbe_platform_with_network_access +build:remote --host_platform=//tools:rbe_platform_with_network_access +build:remote --platforms=//tools:rbe_platform_with_network_access - # Set remote caching settings +# Set remote caching settings build:remote --remote_accept_cached=true # Force remote executions to consider the entire run as linux. diff --git a/.bazelversion b/.bazelversion index 54f6b9be19..fcdb2e109f 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1,3 +1 @@ 4.0.0 -# [NB: this comment has to be after the first line, see https://github.com/bazelbuild/bazelisk/issues/117] -# When updating the Bazel version you also need to update the RBE toolchains version in WORKSPACE diff --git a/WORKSPACE b/WORKSPACE index 01a41ab9b6..6fa10887c5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -41,38 +41,3 @@ yarn_install( strict_visibility = False, # Needed for ts-api-guardian. More info about this can be found https://github.com/bazelbuild/rules_nodejs/wiki#strict_visibility-on-yarn_install-and-npm_install-now-defaults-true-2199 yarn_lock = "//:yarn.lock", ) - -########################## -# Remote Execution Setup # -########################## -# Bring in bazel_toolchains for RBE setup configuration. -http_archive( - name = "bazel_toolchains", - sha256 = "1adf5db506a7e3c465a26988514cfc3971af6d5b3c2218925cd6e71ee443fc3f", - strip_prefix = "bazel-toolchains-4.0.0", - url = "https://github.com/bazelbuild/bazel-toolchains/archive/4.0.0.tar.gz", -) - -load("@bazel_toolchains//rules:environments.bzl", "clang_env") -load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig") - -rbe_autoconfig( - name = "rbe_ubuntu1604_angular", - # Need to specify a base container digest in order to ensure that we can use the checked-in - # platform configurations for the "ubuntu16_04" image. Otherwise the autoconfig rule would - # need to pull the image and run it in order determine the toolchain configuration. See: - # https://github.com/bazelbuild/bazel-toolchains/blob/4.0.0/configs/ubuntu16_04_clang/versions.bzl - base_container_digest = "sha256:f6568d8168b14aafd1b707019927a63c2d37113a03bcee188218f99bd0327ea1", - # Note that if you change the `digest`, you might also need to update the - # `base_container_digest` to make sure marketplace.gcr.io/google/rbe-ubuntu16-04-webtest: - # and marketplace.gcr.io/google/rbe-ubuntu16-04: have - # the same Clang and JDK installed. Clang is needed because of the dependency on - # @com_google_protobuf. Java is needed for the Bazel's test executor Java tool. - digest = "sha256:f743114235a43355bf8324e2ba0fa6a597236fe06f7bc99aaa9ac703631c306b", - env = clang_env(), - registry = "marketplace.gcr.io", - # We can't use the default "ubuntu16_04" RBE image provided by the autoconfig because we need - # a specific Linux kernel that comes with "libx11" in order to run headless browser tests. - repository = "google/rbe-ubuntu16-04-webtest", - use_checked_in_confs = "Force", -) diff --git a/package.json b/package.json index eea32e0420..aa7947a9ee 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,7 @@ "@angular/compiler": "12.0.0-rc.1", "@angular/compiler-cli": "12.0.0-rc.1", "@angular/core": "12.0.0-rc.1", - "@angular/dev-infra-private": "https://github.com/angular/dev-infra-private-builds.git#00bbf3f530406c78cf25d5b37641c5bdac5d047e", + "@angular/dev-infra-private": "https://github.com/angular/dev-infra-private-builds.git#106d8c0d9de461c133e15a21ab6fae93890164f7", "@angular/forms": "12.0.0-rc.1", "@angular/localize": "12.0.0-rc.1", "@angular/material": "11.2.11", diff --git a/renovate.json b/renovate.json index d8440f2963..22693c93a8 100644 --- a/renovate.json +++ b/renovate.json @@ -30,7 +30,7 @@ "pinVersions": false }, { - "packagePatterns": ["^@bazel/.*", "^build_bazel.*", "bazel_toolchains"], + "packagePatterns": ["^@bazel/.*", "^build_bazel.*"], "groupName": "bazel", "pinVersions": false }, diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 379eb2e4f0..23428d243f 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -26,23 +26,11 @@ nodejs_binary( ) platform( - name = "rbe_ubuntu1604-angular", - parents = ["@rbe_ubuntu1604_angular//config:platform"], - remote_execution_properties = """ - {PARENT_REMOTE_EXECUTION_PROPERTIES} - properties: { - name: "dockerAddCapabilities" - value: "SYS_ADMIN" - } - properties: { - name: "dockerNetwork" - value: "standard" - } - properties: { - name: "Pool" - value: "default" - } - """, + name = "rbe_platform_with_network_access", + exec_properties = { + "dockerNetwork": "standard", + }, + parents = ["@npm//@angular/dev-infra-private/bazel/remote-execution:platform"], ) # @external_end diff --git a/yarn.lock b/yarn.lock index abd42f4da5..8c35401715 100644 --- a/yarn.lock +++ b/yarn.lock @@ -84,10 +84,9 @@ dependencies: tslib "^2.0.0" -"@angular/dev-infra-private@https://github.com/angular/dev-infra-private-builds.git#00bbf3f530406c78cf25d5b37641c5bdac5d047e": +"@angular/dev-infra-private@https://github.com/angular/dev-infra-private-builds.git#106d8c0d9de461c133e15a21ab6fae93890164f7": version "0.0.0" - uid "00bbf3f530406c78cf25d5b37641c5bdac5d047e" - resolved "https://github.com/angular/dev-infra-private-builds.git#00bbf3f530406c78cf25d5b37641c5bdac5d047e" + resolved "https://github.com/angular/dev-infra-private-builds.git#106d8c0d9de461c133e15a21ab6fae93890164f7" dependencies: "@angular/benchpress" "0.2.1" "@bazel/buildifier" "^4.0.1" @@ -117,7 +116,7 @@ typed-graphqlify "^3.1.1" typescript "~4.2.4" yaml "^1.10.0" - yargs "^16.2.0" + yargs "^17.0.0" "@angular/forms@12.0.0-rc.1": version "12.0.0-rc.1" @@ -10238,7 +10237,6 @@ sass@1.32.12, sass@^1.32.8: "sauce-connect-proxy@https://saucelabs.com/downloads/sc-4.6.4-linux.tar.gz": version "0.0.0" - uid "992e2cb0d91e54b27a4f5bbd2049f3b774718115" resolved "https://saucelabs.com/downloads/sc-4.6.4-linux.tar.gz#992e2cb0d91e54b27a4f5bbd2049f3b774718115" saucelabs@^1.5.0: @@ -12423,6 +12421,19 @@ yargs@^16.0.0, yargs@^16.1.1, yargs@^16.2.0: y18n "^5.0.5" yargs-parser "^20.2.2" +yargs@^17.0.0: + version "17.0.1" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.0.1.tgz#6a1ced4ed5ee0b388010ba9fd67af83b9362e0bb" + integrity sha512-xBBulfCc8Y6gLFcrPvtqKz9hz8SO0l1Ni8GgDekvBX2ro0HRQImDGnikfc33cgzcYUSncapnNcZDjVFIH3f6KQ== + dependencies: + cliui "^7.0.2" + escalade "^3.1.1" + get-caller-file "^2.0.5" + require-directory "^2.1.1" + string-width "^4.2.0" + y18n "^5.0.5" + yargs-parser "^20.2.2" + yauzl@^2.10.0: version "2.10.0" resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9"