build: perform package.json substitutions in bazel build

.bazelrc

@@ -74,6 +74,8 @@ build:release --stamp
 build:snapshot --workspace_status_command="yarn -s ng-dev release build-env-stamp --mode=snapshot"
 build:snapshot --stamp
 
+build:local --//:enable_package_json_tar_deps
+
 ###############################
 # Output                      #
 ###############################

.prettierignore

@@ -13,3 +13,4 @@
 dist/
 third_party/
 /tests/legacy-cli/e2e/assets/9.0-project/
+/tools/test/*.json

BUILD.bazel

@@ -2,6 +2,8 @@
 #
 # Use of this source code is governed by an MIT-style license that can be
 # found in the LICENSE file at https://angular.io/license
+load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
+
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])
@@ -11,6 +13,7 @@ exports_files([
     "tsconfig.json",  # @external
     "tsconfig-test.json",  # @external
     "tsconfig-build.json",  # @external
+    "package.json",
 ])
 
 # Detect if the build is running under --stamp
@@ -18,3 +21,16 @@ config_setting(
     name = "stamp",
     values = {"stamp": "true"},
 )
+
+# If set will replace dependency versions with tarballs for packages in this repo
+bool_flag(
+    name = "enable_package_json_tar_deps",
+    build_setting_default = False,
+)
+
+config_setting(
+    name = "package_json_use_tar_deps",
+    flag_values = {
+        ":enable_package_json_tar_deps": "true",
+    },
+)

WORKSPACE

@@ -5,6 +5,15 @@ workspace(
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
+http_archive(
+    name = "bazel_skylib",
+    sha256 = "c6966ec828da198c5d9adbaa94c05e3a1c7f21bd012a0b29ba8ddbccb2c93b0d",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz",
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz",
+    ],
+)
+
 http_archive(
     name = "build_bazel_rules_nodejs",
     sha256 = "cfc289523cf1594598215901154a6c2515e8bf3671fd708264a6f6aefe02bf39",
@@ -17,6 +26,10 @@ http_archive(
     urls = ["https://github.com/bazelbuild/rules_pkg/releases/download/0.5.1/rules_pkg-0.5.1.tar.gz"],
 )
 
+load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
+
+bazel_skylib_workspace()
+
 load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
 
 rules_pkg_dependencies()
@@ -50,3 +63,16 @@ yarn_install(
     package_json = "//:package.json",
     yarn_lock = "//:yarn.lock",
 )
+
+http_archive(
+    name = "aspect_bazel_lib",
+    sha256 = "534c9c61b72c257c95302d544984fd8ee63953c233292c5b6952ca5b33cd225e",
+    strip_prefix = "bazel-lib-0.4.2",
+    url = "https://github.com/aspect-build/bazel-lib/archive/v0.4.2.tar.gz",
+)
+
+load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "register_jq_toolchains")
+
+aspect_bazel_lib_dependencies()
+
+register_jq_toolchains(version = "1.6")

packages/angular/cli/BUILD.bazel

@@ -316,7 +316,14 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/architect:package.json",
+        "//packages/angular_devkit/build_angular:package.json",
+        "//packages/angular_devkit/build_webpack:package.json",
+        "//packages/angular_devkit/core:package.json",
+        "//packages/angular_devkit/schematics:package.json",
+        "//packages/schematics/angular:package.json",
+    ],
     deps = [
         ":README.md",
         ":angular-cli",

packages/angular/pwa/BUILD.bazel

@@ -76,7 +76,10 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/schematics:package.json",
+        "//packages/schematics/angular:package.json",
+    ],
     deps = [
         ":README.md",
         ":license",

packages/angular_devkit/architect/BUILD.bazel

@@ -99,7 +99,9 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/core:package.json",
+    ],
     deps = [
         ":README.md",
         ":architect",

packages/angular_devkit/architect_cli/BUILD.bazel

@@ -37,7 +37,10 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/architect:package.json",
+        "//packages/angular_devkit/core:package.json",
+    ],
     deps = [
         ":README.md",
         ":architect_cli",

packages/angular_devkit/benchmark/BUILD.bazel

@@ -78,7 +78,9 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/core:package.json",
+    ],
     deps = [
         "src/test/exit-code-one.js",
         "src/test/fibonacci.js",

packages/angular_devkit/build_angular/BUILD.bazel

@@ -219,7 +219,11 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/architect:package.json",
+        "//packages/angular_devkit/build_webpack:package.json",
+        "//packages/angular_devkit/core:package.json",
+    ],
     deps = [
         ":README.md",
         ":build_angular",

packages/angular_devkit/build_optimizer/BUILD.bazel

@@ -68,7 +68,6 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
     deps = [
         ":README.md",
         ":build_optimizer",

packages/angular_devkit/build_webpack/BUILD.bazel

@@ -109,7 +109,9 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/architect:package.json",
+    ],
     deps = [
         ":README.md",
         ":build_webpack",

packages/angular_devkit/core/BUILD.bazel

@@ -85,7 +85,6 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
     deps = [
         ":README.md",
         ":core",

packages/angular_devkit/schematics/BUILD.bazel

@@ -73,7 +73,9 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/core:package.json",
+    ],
     deps = [
         ":README.md",
         ":collection-schema.json",

packages/angular_devkit/schematics_cli/BUILD.bazel

@@ -98,7 +98,10 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/schematics:package.json",
+        "//packages/angular_devkit/core:package.json",
+    ],
     deps = [
         ":README.md",
         ":license",

packages/ngtools/webpack/BUILD.bazel

@@ -76,7 +76,6 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
     deps = [
         ":README.md",
         ":license",

packages/schematics/angular/BUILD.bazel

@@ -140,7 +140,10 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/schematics:package.json",
+        "//packages/angular_devkit/core:package.json",
+    ],
     deps = [
         "library/library-long.md",
         ":README.md",

scripts/build-bazel.ts

@@ -113,13 +113,6 @@ async function _build(logger: logging.Logger, mode: BuildMode): Promise<string[]
   const queryTargetsCmd = `${bazelCmd} query --output=label "attr(name, npm_package_archive, //packages/...)"`;
   const targets = (await _exec(queryTargetsCmd, true, queryLogger)).split(/\r?\n/);
 
-  let configArg = '';
-  if (mode === 'snapshot') {
-    configArg = '--config=snapshot';
-  } else if (mode === 'release') {
-    configArg = '--config=release';
-  }
-
   const buildLogger = logger.createChild('build');
 
   // If we are in release mode, run `bazel clean` to ensure the execroot and action cache
@@ -132,7 +125,7 @@ async function _build(logger: logging.Logger, mode: BuildMode): Promise<string[]
     await _exec(`${bazelCmd} clean`, false, buildLogger);
   }
 
-  await _exec(`${bazelCmd} build ${configArg} ${targets.join(' ')}`, false, buildLogger);
+  await _exec(`${bazelCmd} build --config=${mode} ${targets.join(' ')}`, false, buildLogger);
 
   return targets;
 }

tools/BUILD.bazel

@@ -7,6 +7,10 @@ load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_binary")
 
 package(default_visibility = ["//visibility:public"])
 
+exports_files([
+    "package_json_release_filter.jq",
+])
+
 nodejs_binary(
     name = "ng_cli_schema",
     data = [

tools/defaults.bzl

@@ -1,9 +1,13 @@
 """Re-export of some bazel rules with repository-wide defaults."""
 
 load("@npm//@bazel/typescript:index.bzl", _ts_library = "ts_library")
-load("@build_bazel_rules_nodejs//:index.bzl", _pkg_npm = "pkg_npm")
+load("@build_bazel_rules_nodejs//:index.bzl", "copy_to_bin", _pkg_npm = "pkg_npm")
 load("@rules_pkg//:pkg.bzl", "pkg_tar")
 load("@npm//@angular/dev-infra-private/bazel:extract_js_module_output.bzl", "extract_js_module_output")
+load("@aspect_bazel_lib//lib:utils.bzl", "to_label")
+load("@aspect_bazel_lib//lib:jq.bzl", "jq")
+load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory")
+load("//tools:link_package_json_to_tarballs.bzl", "link_package_json_to_tarballs")
 
 _DEFAULT_TSCONFIG = "//:tsconfig-build.json"
 _DEFAULT_TSCONFIG_TEST = "//:tsconfig-test.json"
@@ -44,8 +48,20 @@ def ts_library(
         **kwargs
     )
 
-def pkg_npm(name, use_prodmode_output = False, **kwargs):
-    """Default values for pkg_npm"""
+def pkg_npm(name, pkg_deps = [], use_prodmode_output = False, **kwargs):
+    """Override of pkg_npm to produce package outputs and version substitutions conventional to the angular-cli project.
+
+    Produces a package and a tar of that package. Expects a package.json file
+    in the same folder to exist.
+
+    Args:
+        name: Name of the pkg_npm rule. '_archive.tar.gz' is appended to create the tarball.
+        pkg_deps: package.json files of dependent packages. These are used for local path substitutions when --config=local is set.
+        use_prodmode_output: False to ship ES5 devmode output, True to ship ESM output. Defaults to False.
+        **kwargs: Additional arguments passed to the real pkg_npm.
+    """
+    pkg_json = ":package.json"
+
     visibility = kwargs.pop("visibility", None)
 
     common_substitutions = dict(kwargs.pop("substitutions", {}))
@@ -79,6 +95,57 @@ def pkg_npm(name, use_prodmode_output = False, **kwargs):
         deps = deps,
     )
 
+    # Merge package.json with root package.json and perform various substitutions to
+    # prepare it for release. For jq docs, see https://stedolan.github.io/jq/manual/.
+    jq(
+        name = "basic_substitutions",
+        # Note: this jq filter relies on the order of the inputs
+        # buildifier: do not sort
+        srcs = ["//:package.json", pkg_json],
+        filter_file = "//tools:package_json_release_filter.jq",
+        args = ["--slurp"],
+        out = "substituted/package.json",
+    )
+
+    # Copy package.json files to bazel-out so we can use their bazel-out paths to determine
+    # the corresponding package npm package tar.gz path for substitutions.
+    copy_to_bin(
+        name = "package_json_copy",
+        srcs = [pkg_json],
+    )
+    pkg_deps_copies = []
+    for pkg_dep in pkg_deps:
+        pkg_label = to_label(pkg_dep)
+        if pkg_label.name != "package.json":
+            fail("ERROR: only package.json files allowed in pkg_deps of pkg_npm macro")
+        pkg_deps_copies.append("@%s//%s:package_json_copy" % (pkg_label.workspace_name, pkg_label.package))
+
+    # Substitute dependencies on other packages in this repo with tarballs.
+    link_package_json_to_tarballs(
+        name = "tar_substitutions",
+        src = "substituted/package.json",
+        pkg_deps = [":package_json_copy"] + pkg_deps_copies,
+        out = "substituted_with_tars/package.json",
+    )
+
+    # Move the generated package.json along with other deps into a directory for pkg_npm
+    # to package up because pkg_npm requires that all inputs be in the same directory.
+    copy_to_directory(
+        name = "package",
+        srcs = select({
+            # Do tar substitution if config_setting 'package_json_use_tar_deps' is true (local builds)
+            "//:package_json_use_tar_deps": [":%s_js_module_output" % name, "substituted_with_tars/package.json"],
+            "//conditions:default": [":%s_js_module_output" % name, "substituted/package.json"],
+        }),
+        replace_prefixes = {
+            "substituted_with_tars/": "",
+            "substituted/": "",
+        },
+        exclude_prefixes = [
+            "packages",  # Exclude compiled outputs of dependent packages
+        ],
+    )
+
     _pkg_npm(
         name = name,
         # We never set a `package_name` for NPM packages, neither do we enable validation.
@@ -101,7 +168,7 @@ def pkg_npm(name, use_prodmode_output = False, **kwargs):
             "//conditions:default": substitutions,
         }),
         visibility = visibility,
-        deps = [":%s_js_module_output" % name],
+        nested_packages = ["package"],
         tgz = None,
         **kwargs
     )

tools/link_package_json_to_tarballs.bzl

@@ -0,0 +1,87 @@
+# Copyright Google Inc. All Rights Reserved.
+#
+# Use of this source code is governed by an MIT-style license that can be
+# found in the LICENSE file at https://angular.io/license
+load("@aspect_bazel_lib//lib:jq.bzl", "jq")
+load("@aspect_bazel_lib//lib:utils.bzl", "to_label")
+
+def link_package_json_to_tarballs(name, src, pkg_deps, out):
+    """Substitute tar paths into a package.json file for the packages it depends on.
+
+    src and pkg_deps must be labels in the bazel-out tree for the derived path to the npm_package_archive.tar.gz to be correct.
+
+    Args:
+        name: Name of the rule
+        src: package.json file to perform substitions on
+        pkg_deps: package.json files of dependencies to substitute
+        out: Output package.json file
+    """
+
+    src_pkg = to_label(src).package
+
+    # Generate partial jq filters for each dependent package that, when run
+    # against a package.json file, can replace its dependency with a tar path.
+    filter_files = []
+    for i, pkg_dep in enumerate(pkg_deps):
+        pkg_dep_name = "%s_%s.name" % (name, i)
+        pkg_dep_filter = "%s_%s.filter" % (name, i)
+        jq(
+            name = "%s_%s_name" % (name, i),
+            srcs = [pkg_dep],
+            filter = ".name",
+            out = pkg_dep_name,
+        )
+
+        srcs = [
+            pkg_dep_name,
+            pkg_dep,
+        ]
+
+        # Add dependent tars as srcs to include them in the dependency graph, except
+        # for the tar for this package as that would create a circular dependency.
+        pkg_label = to_label(pkg_dep)
+        if pkg_label.package != src_pkg:
+            pkg_tar = "@%s//%s:npm_package_archive.tar.gz" % (pkg_label.workspace_name, pkg_label.package)
+            srcs.append(pkg_tar)
+
+        # Deriving the absolute path to the tar in the execroot requries different
+        # commands depending on whether or not the action is sandboxed.
+        abs_path_sandbox = "readlink $(execpath {pkg_dep})".format(pkg_dep = pkg_dep)
+        abs_path_nosandbox = "(cd $$(dirname $(execpath {pkg_dep})) && pwd)".format(pkg_dep = pkg_dep)
+
+        native.genrule(
+            name = "%s_%s_filter" % (name, i),
+            srcs = srcs,
+            cmd = """
+                TAR=$$(dirname $$({abs_path_sandbox} || {abs_path_nosandbox}))/npm_package_archive.tar.gz
+                PKGNAME=$$(cat $(execpath {pkg_name}))
+                if [[ "$$TAR" != *bazel-out* ]]; then
+                    echo "ERROR: package.json passed to substitute_tar_deps must be in the output tree. You can use copy_to_bin to copy a source file to the output tree."
+                    exit 1
+                fi
+                echo "(..|objects|select(has($${{PKGNAME}})))[$${{PKGNAME}}] |= \\"file:$${{TAR}}\\"" > $@
+            """.format(
+                pkg_name = pkg_dep_name,
+                abs_path_sandbox = abs_path_sandbox,
+                abs_path_nosandbox = abs_path_nosandbox,
+            ),
+            outs = [pkg_dep_filter],
+        )
+        filter_files.append(pkg_dep_filter)
+
+    # Combine all of the filter files into a single filter by joining with |
+    filter = "%s.filter" % name
+    native.genrule(
+        name = "%s_filter" % name,
+        srcs = filter_files,
+        cmd = "cat $(SRCS) | sed '$$!s#$$# |#' > $@",
+        outs = [filter],
+    )
+
+    # Generate final package.json with tar substitutions using the above filter
+    jq(
+        name = name,
+        srcs = [src],
+        filter_file = filter,
+        out = out,
+    )

tools/package_json_release_filter.jq

@@ -0,0 +1,32 @@
+# Copyright Google Inc. All Rights Reserved.
+#
+# Use of this source code is governed by an MIT-style license that can be
+# found in the LICENSE file at https://angular.io/license
+#
+# This filter combines a subproject package.json with the root package.json
+# and performs substitutions to prepare it for release. It should be called
+# with the --slurp argument and be passed the root pacakge.json followed by
+# the subproject package.json.
+#
+# See jq docs for filter syntax: https://stedolan.github.io/jq/manual/.
+
+.[0] as $root
+| .[1] as $proj
+
+# Get the fields from root package.json that should override the project
+# package.json, i.e., every field except the following
+| ($root
+    | del(.bin, .description, .dependencies, .name, .main, .peerDependencies, .optionalDependencies, .typings, .version, .private, .workspaces, .resolutions, .scripts, .["ng-update"])
+) as $root_overrides
+
+# Use the project package.json as a base and override other fields from root
+| $proj + $root_overrides
+
+# Combine keywords from both
+| .keywords = ($root.keywords + $proj.keywords | unique)
+
+# Remove devDependencies
+| del(.devDependencies)
+
+# Add engines
++ {"engines": {"node": "^12.20.0 || ^14.15.0 || >=16.10.0", "npm": "^6.11.0 || ^7.5.6 || >=8.0.0", "yarn": ">= 1.13.0"}}

tools/test/BUILD.bazel

@@ -0,0 +1,32 @@
+load("@bazel_skylib//rules:diff_test.bzl", "diff_test")
+load("@aspect_bazel_lib//lib:jq.bzl", "jq")
+
+jq(
+    name = "final_package_json",
+    # This jq filter relies on the order of the inputs
+    # buildifier: do not sort
+    srcs = [
+        "root_package.json",
+        "project_package.json",
+    ],
+    args = [
+        "--slurp",
+    ],
+    filter_file = "//tools:package_json_release_filter.jq",
+)
+
+# jq outputs CR on windows https://github.com/stedolan/jq/issues/92
+# strip the CRs to do a correct comparison on all platforms
+genrule(
+    name = "final_package_json_cr_stripped",
+    srcs = [":final_package_json"],
+    outs = ["final_package_json_cr_stripped.json"],
+    cmd = "cat $(execpath :final_package_json) | sed \"s#\\r##\" > $@",
+)
+
+# Test correctness of the filter that prepares each project's package.json file for release
+diff_test(
+    name = "package_json_filter_test",
+    file1 = "expected_package.json",
+    file2 = ":final_package_json_cr_stripped",
+)

tools/test/expected_package.json

@@ -0,0 +1,42 @@
+{
+  "name": "project",
+  "version": "0.0.0-SNAPSHOT",
+  "description": "Project package.json",
+  "main": "project/index.js",
+  "bin": {
+    "projectfoo": "./bin/project-foo.js"
+  },
+  "keywords": [
+    "a",
+    "b",
+    "c"
+  ],
+  "scripts": {
+    "build": "node project-build-script"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/angular/angular-cli.git"
+  },
+  "author": "Angular Authors",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/angular/angular-cli/issues"
+  },
+  "homepage": "https://github.com/angular/angular-cli",
+  "dependencies": {
+    "@project/foo": "1.0.0",
+    "@project/bar": "2.0.0"
+  },
+  "ng-update": {
+    "migrations": "@project/migration-collection.json",
+    "packageGroup": {
+      "@project/abc": "0.0.0"
+    }
+  },
+  "engines": {
+    "node": "^12.20.0 || ^14.15.0 || >=16.10.0",
+    "npm": "^6.11.0 || ^7.5.6 || >=8.0.0",
+    "yarn": ">= 1.13.0"
+  }
+}

tools/test/project_package.json

@@ -0,0 +1,39 @@
+{
+  "name": "project",
+  "version": "0.0.0-SNAPSHOT",
+  "description": "Project package.json",
+  "main": "project/index.js",
+  "bin": {
+    "projectfoo": "./bin/project-foo.js"
+  },
+  "keywords": [
+    "b",
+    "c"
+  ],
+  "scripts": {
+    "build": "node project-build-script"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/angular/angular-cli.git"
+  },
+  "author": "Angular Authors",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/angular/angular-cli/issues"
+  },
+  "homepage": "https://github.com/angular/angular-cli",
+  "dependencies": {
+    "@project/foo": "1.0.0",
+    "@project/bar": "2.0.0"
+  },
+  "devDependencies": {
+    "@project/devdep": "1.2.3"
+  },
+  "ng-update": {
+    "migrations": "@project/migration-collection.json",
+    "packageGroup": {
+      "@project/abc": "0.0.0"
+    }
+  }
+}

tools/test/root_package.json

@@ -0,0 +1,37 @@
+{
+  "name": "root",
+  "version": "1.0.0-next.1",
+  "private": true,
+  "description": "Root package.json",
+  "bin": {
+    "root-foo": "./bin/root-foo.js",
+    "root-bar": "./bin/root-bar.js"
+  },
+  "keywords": [
+    "a",
+    "b"
+  ],
+  "scripts": {
+    "build": "node root-build-script"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/angular/angular-cli.git"
+  },
+  "author": "Angular Authors",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/angular/angular-cli/issues"
+  },
+  "homepage": "https://github.com/angular/angular-cli",
+  "workspaces": {
+    "packages": ["packages/root/foo/*", "packages/root/bar/*"]
+  },
+  "resolutions": {
+    "root/foo/bar": "1.0.0"
+  },
+  "devDependencies": {
+    "@root/foo": "1.0.0",
+    "@root/bar": "2.0.0"
+  }
+}