feat(@angular-devkit/benchmark): add package

.monorepo.json

@@ -87,6 +87,11 @@
       "hash": "51b6cc55b25fce87f6a92f0d0942f79c",
       "snapshotRepo": "angular/angular-devkit-architect-cli-builds"
     },
+    "@angular-devkit/benchmark": {
+      "name": "Benchmark",
+      "section": "Tooling",
+      "version": "0.0.0"
+    },
     "@angular-devkit/build-optimizer": {
       "name": "Build Optimizer",
       "links": [

bin/benchmark

@@ -0,0 +1,19 @@
+#!/usr/bin/env node
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+'use strict';
+
+
+require('../lib/bootstrap-local');
+const packages = require('../lib/packages').packages;
+const main = require(packages['@angular-devkit/benchmark'].bin['benchmark']).main;
+
+const args = process.argv.slice(2);
+main({ args })
+  .then(exitCode => process.exitCode = exitCode)
+  .catch(e => { throw (e); });

package.json

@@ -5,6 +5,7 @@
   "description": "Software Development Kit for Angular",
   "bin": {
     "architect": "./bin/architect",
+    "benchmark": "./bin/benchmark",
     "build-optimizer": "./bin/build-optimizer",
     "devkit-admin": "./bin/devkit-admin",
     "ng": "./bin/ng",

packages/angular_devkit/benchmark/BUILD

@@ -0,0 +1,59 @@
+# Copyright Google Inc. All Rights Reserved.
+#
+# Use of this source code is governed by an MIT-style license that can be
+# found in the LICENSE file at https://angular.io/license
+
+licenses(["notice"])  # MIT
+
+load("@build_bazel_rules_typescript//:defs.bzl", "ts_library")
+load("@build_bazel_rules_nodejs//:defs.bzl", "jasmine_node_test")
+
+package(default_visibility = ["//visibility:public"])
+
+ts_library(
+    name = "benchmark",
+    srcs = glob(
+        include = ["src/**/*.ts"],
+        exclude = [
+            "src/**/*_spec.ts",
+            "src/**/*_spec_large.ts",
+            "src/**/*_benchmark.ts",
+        ],
+    ),
+    module_name = "@angular-devkit/benchmark",
+    module_root = "src/index.d.ts",
+    deps = [
+        "//packages/angular_devkit/core",
+        "@rxjs",
+        "@rxjs//operators",
+        # @typings: node
+    ],
+)
+
+ts_library(
+    name = "benchmark_test_lib",
+    srcs = glob(
+        include = [
+            "src/**/*_spec.ts",
+            "src/**/*_spec_large.ts",
+        ],
+    ),
+    data = [
+        "src/test/exit-code-one.js",
+        "src/test/fibonacci.js",
+        "src/test/test-script.js",
+    ],
+    deps = [
+        ":benchmark",
+        "//packages/angular_devkit/core",
+        "@rxjs",
+        "@rxjs//operators",
+        # @typings: jasmine
+        # @typings: node
+    ],
+)
+
+jasmine_node_test(
+    name = "benchmark_test",
+    srcs = [":benchmark_test_lib"],
+)

packages/angular_devkit/benchmark/README.md

@@ -0,0 +1,116 @@
+# Angular Devkit Benchmark
+
+This tool provides benchmark information for processes.
+The tool will gathering metrics from a given command, then average them out over the total runs.
+
+It currently shows only time, process, cpu, and memory used but might be extended in the future.
+
+This tool was created to provide an objective and reproducible way of benchmarking process
+performance.
+
+Given a process (or its source code), process inputs and environment, keeping two of these elements
+constant while varying the third should allow for meaningful benchmarks over time.
+
+In the context of the DevKit, we publish many CLI tools and have access to their source code.
+By tracking tool resource usage we can catch performance regressions or improvements on our CI.
+
+
+## STABILITY AND SUPPORT DISCLAIMER
+
+This package is not currently stable. Usage, output and API may change at any time.
+Support is not ensured.
+
+## Installation
+
+You can install the benchmark tool via `npm install -g benchmark` for a global install, or without
+`-g` for a local install.
+Installing globally gives you access to the `benchmark` binary in your `PATH`.
+
+
+## CLI Usage
+
+Call the `benchmark` binary, followed by options, then double dash, then the command to benchmark.
+
+For more information on the available options, run `benchmark --help`:
+```
+$ benchmark --help
+[benchmark] benchmark [options] -- [command to benchmark]
+
+Collects process stats from running the command.
+
+Options:
+    --help                    Show this message.
+    (... other available options)
+
+Example:
+    benchmark --iterations=3 -- node my-script.js
+```
+
+
+## Example
+
+Given the naive implementation of a fibonacci number calculator below:
+```
+// fibonacci.js
+const fib = (n) => n > 1 ? fib(n - 1) + fib(n - 2) : n;
+console.log(fib(parseInt(process.argv[2])));
+```
+
+Run `benchmark -- node fibonacci.js 40` to benchmark calculating the 40th fibonacci number:
+
+```
+$ benchmark -- node fibonacci.js 40
+[benchmark] Benchmarking process over 5 iterations, with up to 5 retries.
+[benchmark]   node fibonacci.js 40 (at D:\sandbox\latest-project)
+[benchmark] Process Stats
+[benchmark]   Elapsed Time: 2365.40 ms (2449.00, 2444.00, 2357.00, 2312.00, 2265.00)
+[benchmark]   Average Process usage: 1.00 process(es) (1.00, 1.00, 1.00, 1.00, 1.00)
+[benchmark]   Peak Process usage: 1.00 process(es) (1.00, 1.00, 1.00, 1.00, 1.00)
+[benchmark]   Average CPU usage: 4.72 % (5.03, 4.86, 4.50, 4.50, 4.69)
+[benchmark]   Peak CPU usage: 23.40 % (25.00, 23.40, 21.80, 21.80, 25.00)
+[benchmark]   Average Memory usage: 22.34 MB (22.32, 22.34, 22.34, 22.35, 22.35)
+[benchmark]   Peak Memory usage: 22.34 MB (22.32, 22.34, 22.34, 22.35, 22.35)
+```
+
+
+## API Usage
+
+You can also use the benchmarking API directly:
+
+```
+import { Command, defaultStatsCapture, runBenchmark } from '@angular-devkit/benchmark';
+
+const command = new Command('node', ['fibonacci.js', '40']);
+const captures = [defaultStatsCapture];
+
+runBenchmark({ command, command }).subscribe(results => {
+  // results is:[{
+  //   "name": "Process Stats",
+  //   "metrics": [{
+  //     "name": "Elapsed Time", "unit": "ms", "value": 1883.6,
+  //     "componentValues": [1733, 1957, 1580, 1763, 2385]
+  //   }, {
+  //     "name": "Average Process usage", "unit": "process(es)", "value": 1,
+  //     "componentValues": [1, 1, 1, 1, 1]
+  //   }, {
+  //     "name": "Peak Process usage", "unit": "process(es)", "value": 1,
+  //     "componentValues": [1, 1, 1, 1, 1]
+  //   }, {
+  //     "name": "Average CPU usage", "unit": "%", "value": 3.0855555555555556,
+  //     "componentValues": [1.9625, 1.9500000000000002, 1.9500000000000002, 4.887499999999999, 4.677777777777778]
+  //   }, {
+  //     "name": "Peak CPU usage", "unit": "%", "value": 19.380000000000003,
+  //     "componentValues": [15.7, 15.6, 15.6, 25, 25]
+  //   }, {
+  //     "name": "Average Memory usage", "unit": "MB", "value": 22.364057600000002,
+  //     "componentValues": [22.383104, 22.332416, 22.401024, 22.355968, 22.347776]
+  //   }, {
+  //     "name": "Peak Memory usage", "unit": "MB", "value": 22.3649792,
+  //     "componentValues": [22.384639999999997, 22.335487999999998, 22.401024, 22.355968, 22.347776]
+  //   }]
+  // }]
+});
+```
+
+A good example of API usage is the `main` binary itself, found in `./src/main.ts`.
+We recommend using TypeScript to get full access to the interfaces included.

packages/angular_devkit/benchmark/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@angular-devkit/benchmark",
+  "version": "0.0.0",
+  "private": true,
+  "description": "Angular Benchmark",
+  "bin": {
+    "benchmark": "./src/main.js"
+  },
+  "keywords": [
+    "benchmark"
+  ],
+  "engines": {
+    "node": ">= 8.9.0",
+    "npm": ">= 5.5.1"
+  },
+  "dependencies": {
+    "@angular-devkit/core": "0.0.0",
+    "minimist": "^1.2.0",
+    "pidusage": "^2.0.16",
+    "pidtree": "^0.3.0",
+    "rxjs": "~6.2.0",
+    "tree-kill": "^1.2.0"
+  }
+}

packages/angular_devkit/benchmark/src/command.ts

@@ -0,0 +1,23 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+export class Command {
+  constructor(
+    public cmd: string,
+    public args: string[] = [],
+    public cwd: string = process.cwd(),
+    public expectedExitCode = 0,
+  ) { }
+
+  toString() {
+    const { cmd, args, cwd } = this;
+    const argsStr = args.length > 0 ? ' ' + args.join(' ') : '';
+
+    return `${cmd}${argsStr} (at ${cwd})`;
+  }
+}

packages/angular_devkit/benchmark/src/default-reporter.ts

@@ -0,0 +1,24 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import { logging, tags } from '@angular-devkit/core';
+import { AggregatedMetric, BenchmarkReporter, Metric } from './interfaces';
+
+export const defaultReporter = (logger: logging.Logger): BenchmarkReporter => (process, groups) => {
+  const toplevelLogger = logger;
+  const indentLogger = new logging.IndentLogger('benchmark-indent-logger', toplevelLogger);
+
+  const formatMetric = (metric: Metric | AggregatedMetric) => tags.oneLine`
+    ${metric.name}: ${metric.value.toFixed(2)} ${metric.unit}
+    ${metric.componentValues ? `(${metric.componentValues.map(v => v.toFixed(2)).join(', ')})` : ''}
+  `;
+
+  groups.forEach(group => {
+    toplevelLogger.info(`${group.name}`);
+    group.metrics.forEach(metric => indentLogger.info(formatMetric(metric)));
+  });
+};

packages/angular_devkit/benchmark/src/default-stats-capture.ts

@@ -0,0 +1,60 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import { Observable } from 'rxjs';
+import { map, reduce } from 'rxjs/operators';
+import { AggregatedProcessStats, Capture, MetricGroup, MonitoredProcess } from './interfaces';
+import { cumulativeMovingAverage, max } from './utils';
+
+
+export const defaultStatsCapture: Capture = (
+  process: MonitoredProcess,
+): Observable<MetricGroup> => {
+  type Accumulator = {
+    elapsed: number,
+    avgProcesses: number,
+    peakProcesses: number,
+    avgCpu: number,
+    peakCpu: number,
+    avgMemory: number,
+    peakMemory: number,
+  };
+  const seed: Accumulator = {
+    elapsed: 0,
+    avgProcesses: 0,
+    peakProcesses: 0,
+    avgCpu: 0,
+    peakCpu: 0,
+    avgMemory: 0,
+    peakMemory: 0,
+  };
+
+  return process.stats$.pipe(
+    reduce<AggregatedProcessStats, Accumulator>((acc, val, idx) => ({
+      elapsed: val.elapsed,
+      avgProcesses: cumulativeMovingAverage(acc.avgProcesses, val.processes, idx),
+      peakProcesses: max(acc.peakProcesses, val.processes),
+      avgCpu: cumulativeMovingAverage(acc.avgCpu, val.cpu, idx),
+      peakCpu: max(acc.peakCpu, val.cpu),
+      avgMemory: cumulativeMovingAverage(acc.avgMemory, val.memory, idx),
+      peakMemory: max(acc.peakMemory, val.memory),
+    }), seed),
+    map(metrics => ({
+      name: 'Process Stats',
+      metrics: [
+        { name: 'Elapsed Time', unit: 'ms', value: metrics.elapsed },
+        { name: 'Average Process usage', unit: 'process(es)', value: metrics.avgProcesses },
+        { name: 'Peak Process usage', unit: 'process(es)', value: metrics.peakProcesses },
+        { name: 'Average CPU usage', unit: '%', value: metrics.avgCpu },
+        { name: 'Peak CPU usage', unit: '%', value: metrics.peakCpu },
+        { name: 'Average Memory usage', unit: 'MB', value: metrics.avgMemory * 1e-6 },
+        { name: 'Peak Memory usage', unit: 'MB', value: metrics.peakMemory * 1e-6 },
+      ],
+    })),
+  );
+};

packages/angular_devkit/benchmark/src/default-stats-capture_spec.ts

@@ -0,0 +1,48 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import { Observable } from 'rxjs';
+import { toArray } from 'rxjs/operators';
+import { defaultStatsCapture } from './default-stats-capture';
+import { AggregatedProcessStats, MonitoredProcess } from './interfaces';
+
+
+describe('defaultStatsCapture', () => {
+  it('works', async () => {
+    const stats$ = new Observable<AggregatedProcessStats>(obs => {
+      const ignoredStats = { ppid: 1, pid: 1, ctime: 1, timestamp: 1 };
+      obs.next({
+        processes: 1, cpu: 0, memory: 10 * 1e6, elapsed: 1000,
+        ...ignoredStats,
+      });
+      obs.next({
+        processes: 3, cpu: 40, memory: 2 * 1e6, elapsed: 2000,
+        ...ignoredStats,
+      });
+      obs.next({
+        processes: 5, cpu: 20, memory: 3 * 1e6, elapsed: 3000,
+        ...ignoredStats,
+      });
+      obs.complete();
+    });
+    const process = { stats$ } as {} as MonitoredProcess;
+
+    const res = await defaultStatsCapture(process).pipe(toArray()).toPromise();
+    expect(res).toEqual([{
+      name: 'Process Stats',
+      metrics: [
+        { name: 'Elapsed Time', unit: 'ms', value: 3000 },
+        { name: 'Average Process usage', unit: 'process(es)', value: 3 },
+        { name: 'Peak Process usage', unit: 'process(es)', value: 5 },
+        { name: 'Average CPU usage', unit: '%', value: 20 },
+        { name: 'Peak CPU usage', unit: '%', value: 40 },
+        { name: 'Average Memory usage', unit: 'MB', value: 5 },
+        { name: 'Peak Memory usage', unit: 'MB', value: 10 },
+      ],
+    }]);
+  });
+});

packages/angular_devkit/benchmark/src/index.ts

@@ -0,0 +1,16 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+export * from './interfaces';
+export * from './command';
+export * from './default-reporter';
+export * from './default-stats-capture';
+export * from './monitored-process';
+export * from './run-benchmark';
+export * from './utils';
+export * from './main';

packages/angular_devkit/benchmark/src/interfaces.d.ts

@@ -0,0 +1,49 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import { Observable } from 'rxjs';
+import { Command } from './command';
+
+export interface AggregatedProcessStats {
+  processes: number;      // number of processes
+  cpu: number;            // percentage (from 0 to 100*vcore)
+  memory: number;         // bytes
+  ppid: number;           // PPID
+  pid: number;            // PID
+  ctime: number;          // ms user + system time
+  elapsed: number;        // ms since the start of the process
+  timestamp: number;      // ms since epoch
+}
+
+export interface MonitoredProcess {
+  stats$: Observable<AggregatedProcessStats>;
+  stdout$: Observable<Buffer>;
+  stderr$: Observable<Buffer>;
+  run(): Observable<number>;
+  toString(): string;
+}
+
+export interface Metric {
+  name: string;
+  unit: string;
+  value: number;
+  componentValues?: number[];
+}
+
+export interface AggregatedMetric extends Metric {
+  componentValues: number[];
+}
+
+export interface MetricGroup {
+  name: string;
+  metrics: (Metric | AggregatedMetric)[];
+}
+
+export type Capture = (process: MonitoredProcess) => Observable<MetricGroup>;
+
+// TODO: might need to allow reporters to say they are finished.
+export type BenchmarkReporter = (command: Command, groups: MetricGroup[]) => void;

packages/angular_devkit/benchmark/src/main.ts

@@ -0,0 +1,184 @@
+#!/usr/bin/env node
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import { logging, tags, terminal } from '@angular-devkit/core';
+import { appendFileSync, writeFileSync } from 'fs';
+import * as minimist from 'minimist';
+import { filter, map, toArray } from 'rxjs/operators';
+import { Command } from '../src/command';
+import { defaultReporter } from '../src/default-reporter';
+import { defaultStatsCapture } from '../src/default-stats-capture';
+import { runBenchmark } from '../src/run-benchmark';
+
+
+export interface MainOptions {
+  args: string[];
+  stdout?: { write(buffer: string | Buffer): boolean };
+  stderr?: { write(buffer: string | Buffer): boolean };
+}
+
+export async function main({
+  args,
+  stdout = process.stdout,
+  stderr = process.stderr,
+}: MainOptions): Promise<0 | 1> {
+
+  // Show usage of the CLI tool, and exit the process.
+  function usage(logger: logging.Logger) {
+    logger.info(tags.stripIndent`
+    benchmark [options] -- [command to benchmark]
+
+    Collects process stats from running the command.
+
+    Options:
+        --help                    Show this message.
+        --verbose                 Show more information while running.
+        --exit-code               Expected exit code for the command. Default is 0.
+        --iterations              Number of iterations to run the benchmark over. Default is 5.
+        --retries                 Number of times to retry when process fails. Default is 5.
+        --cwd                     Current working directory to run the process in.
+        --output-file             File to output benchmark log to.
+        --overwrite-output-file   If the output file should be overwritten rather than appended to.
+        --prefix                  Logging prefix.
+
+    Example:
+        benchmark --iterations=3 -- node my-script.js
+  `);
+  }
+
+  interface BenchmarkCliArgv {
+    help: boolean;
+    verbose: boolean;
+    'overwrite-output-file': boolean;
+    'exit-code': number;
+    iterations: number;
+    retries: number;
+    'output-file': string | null;
+    cwd: string;
+    prefix: string;
+    '--': string[] | null;
+  }
+
+  // Parse the command line.
+  const argv = minimist(args, {
+    boolean: ['help', 'verbose', 'overwrite-output-file'],
+    default: {
+      'exit-code': 0,
+      'iterations': 5,
+      'retries': 5,
+      'output-file': null,
+      'cwd': process.cwd(),
+      'prefix': '[benchmark]',
+    },
+    '--': true,
+  }) as {} as BenchmarkCliArgv;
+
+  // Create the DevKit Logger used through the CLI.
+  const logger = new logging.TransformLogger(
+    'benchmark-prefix-logger',
+    stream => stream.pipe(map(entry => {
+      if (argv['prefix']) { entry.message = `${argv['prefix']} ${entry.message}`; }
+
+      return entry;
+    })),
+  );
+
+  // Log to console.
+  logger
+    .pipe(filter(entry => (entry.level != 'debug' || argv['verbose'])))
+    .subscribe(entry => {
+      let color: (s: string) => string = x => terminal.dim(terminal.white(x));
+      let output = stdout;
+      switch (entry.level) {
+        case 'info':
+          color = terminal.white;
+          break;
+        case 'warn':
+          color = terminal.yellow;
+          break;
+        case 'error':
+          color = terminal.red;
+          output = stderr;
+          break;
+        case 'fatal':
+          color = (x: string) => terminal.bold(terminal.red(x));
+          output = stderr;
+          break;
+      }
+
+      output.write(color(entry.message) + '\n');
+    });
+
+
+  // Print help.
+  if (argv['help']) {
+    usage(logger);
+
+    return 0;
+  }
+
+  const commandArgv = argv['--'];
+
+  // Exit early if we can't find the command to benchmark.
+  if (!commandArgv || !Array.isArray(argv['--']) || (argv['--'] as Array<string>).length < 1) {
+    logger.fatal(`Missing command, see benchmark --help for help.`);
+
+    return 1;
+  }
+
+  // Setup file logging.
+  if (argv['output-file'] !== null) {
+    if (argv['overwrite-output-file']) {
+      writeFileSync(argv['output-file'] as string, '');
+    }
+    logger.pipe(filter(entry => (entry.level != 'debug' || argv['verbose'])))
+      .subscribe(entry => appendFileSync(argv['output-file'] as string, `${entry.message}\n`));
+  }
+
+  // Run benchmark on given command, capturing stats and reporting them.
+  const exitCode = argv['exit-code'];
+  const cmd = commandArgv[0];
+  const cmdArgs = commandArgv.slice(1);
+  const command = new Command(cmd, cmdArgs, argv['cwd'], exitCode);
+  const captures = [defaultStatsCapture];
+  const reporters = [defaultReporter(logger)];
+  const iterations = argv['iterations'];
+  const retries = argv['retries'];
+
+  logger.info(`Benchmarking process over ${iterations} iterations, with up to ${retries} retries.`);
+  logger.info(`  ${command.toString()}`);
+
+  let res;
+  try {
+    res = await runBenchmark(
+      { command, captures, reporters, iterations, retries, logger },
+    ).pipe(toArray()).toPromise();
+  } catch (error) {
+    if (error.message) {
+      logger.fatal(error.message);
+    } else {
+      logger.fatal(error);
+    }
+
+    return 1;
+  }
+
+  if (res.length === 0) {
+    return 1;
+  }
+
+  return 0;
+}
+
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  main({ args })
+    .then(exitCode => process.exitCode = exitCode)
+    .catch(e => { throw (e); });
+}

packages/angular_devkit/benchmark/src/main_spec.ts

@@ -0,0 +1,145 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'fs';
+import { basename, dirname, join } from 'path';
+import { main } from './main';
+// tslint:disable-next-line:no-implicit-dependencies
+const temp = require('temp');
+
+
+// We only care about the write method in these mocks of NodeJS.WriteStream.
+class MockWriteStream {
+  lines: string[] = [];
+  write(str: string) {
+    // Strip color control characters.
+    this.lines.push(str.replace(/[^\x20-\x7F]\[\d+m/g, ''));
+
+    return true;
+  }
+}
+
+describe('benchmark binary', () => {
+  const benchmarkScript = require.resolve(join(__dirname, './test/fibonacci.js'));
+  const exitCodeOneScript = require.resolve(join(__dirname, './test/exit-code-one.js'));
+  const outputFileRoot = temp.mkdirSync('benchmark-binary-spec-');
+  const outputFile = join(outputFileRoot, 'output.log');
+  let stdout: MockWriteStream, stderr: MockWriteStream;
+
+  beforeEach(() => {
+    stdout = new MockWriteStream();
+    stderr = new MockWriteStream();
+  });
+
+  afterEach(() => {
+    if (existsSync(outputFile)) { unlinkSync(outputFile); }
+  });
+
+  it('works', async () => {
+    const args = ['--', 'node', benchmarkScript, '30'];
+    const res = await main({ args, stdout, stderr });
+    expect(stdout.lines).toContain('[benchmark] Process Stats\n');
+    expect(res).toEqual(0);
+  });
+
+  it('fails with no command', async () => {
+    const args: string[] = [];
+    const res = await main({ args, stdout, stderr });
+    expect(stderr.lines).toContain(
+      '[benchmark] Missing command, see benchmark --help for help.\n');
+    expect(res).toEqual(1);
+  });
+
+  it('fails with when exit code is not expected', async () => {
+    const args: string[] = ['--', 'node', exitCodeOneScript];
+    const res = await main({ args, stdout, stderr });
+    expect(stderr.lines).toContain(
+      '[benchmark] Maximum number of retries (5) for command was exceeded.\n');
+    expect(res).toEqual(1);
+  });
+
+  it('prints help', async () => {
+    const args = ['--help'];
+    const res = await main({ args, stdout, stderr });
+    // help is a multiline write.
+    expect(stdout.lines[0]).toContain('Options:\n');
+    expect(res).toEqual(0);
+  });
+
+  it('uses verbose', async () => {
+    const args = ['--verbose', '--', 'node', benchmarkScript, '30'];
+    const res = await main({ args, stdout, stderr });
+    expect(stdout.lines).toContain('[benchmark] Run #1: finished successfully\n');
+    expect(res).toEqual(0);
+  });
+
+  it('uses exit code', async () => {
+    const args = ['--exit-code', '1', '--', 'node', exitCodeOneScript];
+    const res = await main({ args, stdout, stderr });
+    expect(stdout.lines).toContain('[benchmark] Process Stats\n');
+    expect(res).toEqual(0);
+  });
+
+  it('uses iterations', async () => {
+    const args = ['--iterations', '3', '--', 'node', benchmarkScript, '30'];
+    const res = await main({ args, stdout, stderr });
+    expect(stdout.lines).toContain(
+      '[benchmark] Benchmarking process over 3 iterations, with up to 5 retries.\n');
+    expect(res).toEqual(0);
+  });
+
+  it('uses retries', async () => {
+    const args = ['--retries', '3', '--', 'node', benchmarkScript, '30'];
+    const res = await main({ args, stdout, stderr });
+    expect(stdout.lines).toContain(
+      '[benchmark] Benchmarking process over 5 iterations, with up to 3 retries.\n');
+    expect(res).toEqual(0);
+  });
+
+  it('uses cwd', async () => {
+    const args = ['--cwd', dirname(benchmarkScript), '--', 'node', basename(benchmarkScript), '30'];
+    const res = await main({ args, stdout, stderr });
+    expect(stdout.lines).toContain('[benchmark] Process Stats\n');
+    expect(res).toEqual(0);
+  });
+
+  it('uses output-file', async () => {
+    const args = ['--output-file', outputFile, '--', 'node', benchmarkScript, '30'];
+    const res = await main({ args, stdout, stderr });
+    expect(res).toEqual(0);
+    expect(existsSync(outputFile)).toBe(true, 'outputFile exists');
+    expect(readFileSync(outputFile, 'utf-8')).toContain('[benchmark] Process Stats');
+  });
+
+  it('appends to output-file', async () => {
+    writeFileSync(outputFile, 'existing line');
+    const args = ['--output-file', outputFile, '--', 'node', benchmarkScript, '30'];
+    const res = await main({ args, stdout, stderr });
+    expect(res).toEqual(0);
+    expect(existsSync(outputFile)).toBe(true, 'outputFile exists');
+    expect(readFileSync(outputFile, 'utf-8')).toContain('existing line');
+  });
+
+  it('overwrites output-file', async () => {
+    writeFileSync(outputFile, 'existing line');
+    const args = [
+      '--output-file', outputFile, '--overwrite-output-file',
+      '--', 'node', benchmarkScript, '30',
+    ];
+    const res = await main({ args, stdout, stderr });
+    expect(res).toEqual(0);
+    expect(existsSync(outputFile)).toBe(true, 'outputFile exists');
+    expect(readFileSync(outputFile, 'utf-8')).not.toContain('existing line');
+  });
+
+  it('uses prefix', async () => {
+    const args = ['--prefix', '[abc]', '--', 'node', benchmarkScript, '30'];
+    const res = await main({ args, stdout, stderr });
+    stdout.lines.forEach(line => expect(line).toMatch(/^\[abc\]/));
+    expect(res).toEqual(0);
+  });
+});

packages/angular_devkit/benchmark/src/monitored-process.ts

@@ -0,0 +1,106 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import { SpawnOptions, spawn } from 'child_process';
+import { Observable, Subject, from, timer } from 'rxjs';
+import { concatMap, map, onErrorResumeNext, tap } from 'rxjs/operators';
+import { Command } from './command';
+import { AggregatedProcessStats, MonitoredProcess } from './interfaces';
+const pidusage = require('pidusage');
+const pidtree = require('pidtree');
+const treeKill = require('tree-kill');
+
+
+// Cleanup when the parent process exits.
+const defaultProcessExitCb = () => { };
+let processExitCb = defaultProcessExitCb;
+process.on('exit', () => {
+  processExitCb();
+  processExitCb = defaultProcessExitCb;
+});
+
+export class LocalMonitoredProcess implements MonitoredProcess {
+  private stats = new Subject<AggregatedProcessStats>();
+  private stdout = new Subject<Buffer>();
+  private stderr = new Subject<Buffer>();
+  private pollingRate = 100;
+  stats$: Observable<AggregatedProcessStats> = this.stats.asObservable();
+  stdout$: Observable<Buffer> = this.stdout.asObservable();
+  stderr$: Observable<Buffer> = this.stderr.asObservable();
+
+  constructor(private command: Command) { }
+
+  run(): Observable<number> {
+    return new Observable(obs => {
+      const { cmd, cwd, args } = this.command;
+      const spawnOptions: SpawnOptions = { cwd: cwd, shell: true };
+
+      // Spawn the process.
+      const childProcess = spawn(cmd, args, spawnOptions);
+
+      // Emit output and stats.
+      childProcess.stdout.on('data', (data: Buffer) => this.stdout.next(data));
+      childProcess.stderr.on('data', (data: Buffer) => this.stderr.next(data));
+      const statsSubs = timer(0, this.pollingRate).pipe(
+        concatMap(() => from(pidtree(childProcess.pid, { root: true }))),
+        concatMap((pids: number[]) => from(pidusage(pids, { maxage: 5 * this.pollingRate }))),
+        map((statsByProcess: { [key: string]: AggregatedProcessStats }) => {
+          // Ignore the spawned shell in the total process number.
+          const pids = Object.keys(statsByProcess)
+            .filter(pid => pid != childProcess.pid.toString());
+          const processes = pids.length;
+          // We want most stats from the parent process.
+          const { pid, ppid, ctime, elapsed, timestamp } = statsByProcess[childProcess.pid];
+          // CPU and memory should be agreggated.
+          let cpu = 0, memory = 0;
+          for (const pid of pids) {
+            cpu += statsByProcess[pid].cpu;
+            memory += statsByProcess[pid].memory;
+          }
+
+          return {
+            processes, cpu, memory, pid, ppid, ctime, elapsed, timestamp,
+          } as AggregatedProcessStats;
+        }),
+        tap(stats => this.stats.next(stats)),
+        onErrorResumeNext(),
+      ).subscribe();
+
+      // Process event handling.
+
+      // Killing processes cross platform can be hard, treeKill helps.
+      const killChildProcess = () => {
+        if (childProcess && childProcess.pid) {
+          treeKill(childProcess.pid, 'SIGTERM');
+        }
+      };
+
+      // Convert process exit codes and errors into observable events.
+      const handleChildProcessExit = (code?: number, error?: Error) => {
+        // Stop gathering stats and complete subjects.
+        statsSubs.unsubscribe();
+        this.stats.complete();
+        this.stdout.complete();
+        this.stderr.complete();
+
+        // Kill hanging child processes and emit error/exit code.
+        killChildProcess();
+        if (error) {
+          obs.error(error);
+        }
+        obs.next(code);
+        obs.complete();
+      };
+      childProcess.once('exit', handleChildProcessExit);
+      childProcess.once('error', (err) => handleChildProcessExit(1, err));
+      processExitCb = killChildProcess;
+
+      // Cleanup on unsubscription.
+      return () => childProcess.kill();
+    });
+  }
+}

packages/angular_devkit/benchmark/src/monitored-process_spec.ts

@@ -0,0 +1,50 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import { dirname } from 'path';
+import { toArray } from 'rxjs/operators';
+import { Command } from './command';
+import { LocalMonitoredProcess } from './monitored-process';
+
+describe('LocalMonitoredProcess', () => {
+  const cmd = new Command(
+    'node',
+    ['test-script.js'],
+    dirname(require.resolve('./test/test-script.js')),
+  );
+
+  it('works', async () => {
+    const process = new LocalMonitoredProcess(cmd);
+    const res = await process.run().pipe(toArray()).toPromise();
+    expect(res).toEqual([0]);
+  });
+
+  it('captures stdout', async () => {
+    const process = new LocalMonitoredProcess(cmd);
+    const stdoutOutput: string[] = [];
+    process.stdout$.subscribe(data => stdoutOutput.push(data.toString()));
+    await process.run().pipe().toPromise();
+    expect(stdoutOutput).toEqual(['stdout start\n', 'stdout end\n']);
+  });
+
+  it('captures stderr', async () => {
+    const process = new LocalMonitoredProcess(cmd);
+    const stdoutOutput: string[] = [];
+    process.stderr$.subscribe(data => stdoutOutput.push(data.toString()));
+    await process.run().pipe().toPromise();
+    expect(stdoutOutput).toEqual(['stderr start\n', 'stderr end\n']);
+  });
+
+  it('captures stats', async () => {
+    const process = new LocalMonitoredProcess(cmd);
+    const statsOutput: string[] = [];
+    process.stderr$.subscribe(data => statsOutput.push(data.toString()));
+    await process.run().pipe().toPromise();
+    expect(statsOutput.length).toBeGreaterThan(0);
+  });
+});

packages/angular_devkit/benchmark/src/run-benchmark.ts

@@ -0,0 +1,98 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import { BaseException, logging } from '@angular-devkit/core';
+import { Observable, forkJoin, of, throwError } from 'rxjs';
+import { concatMap, map, retryWhen, take, tap, throwIfEmpty } from 'rxjs/operators';
+import { Command } from './command';
+import { BenchmarkReporter, Capture, MetricGroup } from './interfaces';
+import { LocalMonitoredProcess } from './monitored-process';
+import { aggregateMetricGroups } from './utils';
+
+export interface RunBenchmarkOptions {
+  command: Command;
+  captures: Capture[];
+  reporters: BenchmarkReporter[];
+  iterations?: number;
+  retries?: number;
+  expectedExitCode?: number;
+  logger?: logging.Logger;
+}
+
+export class MaximumRetriesExceeded extends BaseException {
+  constructor(retries: number) {
+    super(`Maximum number of retries (${retries}) for command was exceeded.`);
+  }
+}
+
+export function runBenchmark({
+  command, captures, reporters = [], iterations = 5, retries = 5, logger = new logging.NullLogger(),
+}: RunBenchmarkOptions): Observable<MetricGroup[]> {
+
+  let successfulRuns = 0;
+  let failedRuns = 0;
+  const notDoneYet = new BaseException('Not done yet.');
+  const processFailed = new BaseException('Wrong exit code.');
+  const debugPrefix = () => `Run #${successfulRuns + 1}:`;
+  let aggregatedMetricGroups: MetricGroup[] = [];
+
+  // Run the process and captures, wait for both to finish, and average out the metrics.
+  return new Observable(obs => {
+    const monitoredProcess = new LocalMonitoredProcess(command);
+    const metric$ = captures.map(capture => capture(monitoredProcess));
+    obs.next([monitoredProcess, ...metric$]);
+  }).pipe(
+    tap(() => logger.debug(`${debugPrefix()} starting`)),
+    concatMap(([monitoredProcess, ...metric$]) => forkJoin(monitoredProcess.run(), ...metric$)),
+    throwIfEmpty(() => new Error('Nothing was captured')),
+    concatMap((results) => {
+      const [processExitCode, ...metrics] = results;
+
+      if ((processExitCode as number) != command.expectedExitCode) {
+        logger.debug(`${debugPrefix()} exited with ${processExitCode} but `
+          + `${command.expectedExitCode} was expected`);
+
+        return throwError(processFailed);
+      }
+
+      logger.debug(`${debugPrefix()} finished successfully`);
+
+      return of(metrics as MetricGroup[]);
+    }),
+    map(newMetricGroups => {
+      // Aggregate metric groups into a single one.
+      if (aggregatedMetricGroups.length === 0) {
+        aggregatedMetricGroups = newMetricGroups;
+      } else {
+        aggregatedMetricGroups = aggregatedMetricGroups.map((_, idx) =>
+          aggregateMetricGroups(aggregatedMetricGroups[idx], newMetricGroups[idx]),
+        );
+      }
+
+      successfulRuns += 1;
+
+      return aggregatedMetricGroups;
+    }),
+    concatMap(val => successfulRuns < iterations ? throwError(notDoneYet) : of(val)),
+    // This is where we control when the process should be run again.
+    retryWhen(errors => errors.pipe(concatMap(val => {
+      // Always run again while we are not done yet.
+      if (val === notDoneYet) { return of(val); }
+
+      // Otherwise check if we're still within the retry threshold.
+      failedRuns += 1;
+      if (failedRuns < retries) { return of(val); }
+
+      if (val === processFailed) { return throwError(new MaximumRetriesExceeded(retries)); }
+
+      // Not really sure what happened here, just re-throw it.
+      return throwError(val);
+    }))),
+    tap(groups => reporters.forEach(reporter => reporter(command, groups))),
+    take(1),
+  );
+}

packages/angular_devkit/benchmark/src/test/exit-code-one.js

@@ -0,0 +1 @@
+process.exit(1);

packages/angular_devkit/benchmark/src/test/fibonacci.js

@@ -0,0 +1,2 @@
+const fib = (n) => n > 1 ? fib(n - 1) + fib(n - 2) : n;
+console.log(fib(parseInt(process.argv[2])));

packages/angular_devkit/benchmark/src/test/test-script.js

@@ -0,0 +1,7 @@
+console.log("stdout start");
+console.error("stderr start");
+
+setTimeout(() => {
+  console.log("stdout end");
+  console.error("stderr end");
+}, 1000);

packages/angular_devkit/benchmark/src/utils.ts

@@ -0,0 +1,49 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import { AggregatedMetric, Metric, MetricGroup } from './interfaces';
+
+// Prefers to keep v1 when both are equal.
+export const max = (v1: number, v2: number) => v2 > v1 ? v2 : v1;
+
+export const cumulativeMovingAverage = (acc: number, val: number, accSize: number) =>
+  (val + accSize * acc) / (accSize + 1);
+
+export const aggregateMetrics = (
+  m1: Metric | AggregatedMetric,
+  m2: Metric | AggregatedMetric,
+): AggregatedMetric => {
+  if ((m1.name != m2.name) || (m1.unit != m2.unit)) {
+    throw new Error('Cannot aggregate metrics with different names or units:');
+  }
+
+  const m1Values = m1.componentValues ? m1.componentValues : [m1.value];
+  const m2Values = m2.componentValues ? m2.componentValues : [m2.value];
+
+  return {
+    name: m1.name,
+    unit: m1.unit,
+    // m1.value already holds an average if it has component values.
+    value: m2Values.reduce(
+      (acc, val, idx) => cumulativeMovingAverage(acc, val, idx + m1Values.length),
+      m1.value,
+    ),
+    componentValues: [...m1Values, ...m2Values],
+  };
+};
+
+export const aggregateMetricGroups = (g1: MetricGroup, g2: MetricGroup): MetricGroup => {
+  if (g1.name != g2.name || g1.metrics.length != g2.metrics.length) {
+    throw new Error('Cannot aggregate metric groups with different names.');
+  }
+
+  return {
+    name: g1.name,
+    metrics: g1.metrics.map((_, idx) => aggregateMetrics(g1.metrics[idx], g2.metrics[idx])),
+  };
+};

yarn.lock

@@ -5719,6 +5719,16 @@ performance-now@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
 
+pidtree@^0.3.0:
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/pidtree/-/pidtree-0.3.0.tgz#f6fada10fccc9f99bf50e90d0b23d72c9ebc2e6b"
+
+pidusage@^2.0.16:
+  version "2.0.16"
+  resolved "https://registry.yarnpkg.com/pidusage/-/pidusage-2.0.16.tgz#46e2e3185eaef253ef6303f766f7aae72f74f98c"
+  dependencies:
+    safe-buffer "^5.1.2"
+
 pify@^2.0.0, pify@^2.3.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c"