test: run tests in isolated subprocess

tests/legacy-cli/BUILD.bazel

@@ -20,6 +20,7 @@ ts_library(
 
         # Loaded dynamically at runtime, not compiletime deps
         "//tests/legacy-cli/e2e/setup",
+        "//tests/legacy-cli/e2e/initialize",
         "//tests/legacy-cli/e2e/tests",
     ],
 )

tests/legacy-cli/e2e/initialize/500-create-project.ts

@@ -1,13 +1,14 @@
 import { join } from 'path';
+import yargsParser from 'yargs-parser';
 import { getGlobalVariable } from '../utils/env';
 import { expectFileToExist } from '../utils/fs';
 import { gitClean } from '../utils/git';
-import { setRegistry as setNPMConfigRegistry } from '../utils/packages';
+import { installPackage, setRegistry as setNPMConfigRegistry } from '../utils/packages';
 import { ng } from '../utils/process';
 import { prepareProjectForE2e, updateJsonFile } from '../utils/project';
 
 export default async function () {
-  const argv = getGlobalVariable('argv');
+  const argv = getGlobalVariable<yargsParser.Arguments>('argv');
 
   if (argv.noproject) {
     return;
@@ -20,6 +21,14 @@ export default async function () {
     // Ensure local test registry is used when outside a project
     await setNPMConfigRegistry(true);
 
+    // Install puppeteer in the parent directory for use by the CLI within any test project.
+    // Align the version with the primary project package.json.
+    const puppeteerVersion = require('../../../../package.json').devDependencies.puppeteer.replace(
+      /^[\^~]/,
+      '',
+    );
+    await installPackage(`puppeteer@${puppeteerVersion}`);
+
     await ng('new', 'test-project', '--skip-install');
     await expectFileToExist(join(process.cwd(), 'test-project'));
     process.chdir('./test-project');

tests/legacy-cli/e2e/initialize/BUILD.bazel

@@ -0,0 +1,15 @@
+load("//tools:defaults.bzl", "ts_library")
+
+ts_library(
+    name = "initialize",
+    testonly = True,
+    srcs = glob(["**/*.ts"]),
+    data = [
+        "//:package.json",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tests/legacy-cli/e2e/utils",
+        "@npm//@types/yargs-parser",
+    ],
+)

tests/legacy-cli/e2e/setup/002-npm-sandbox.ts

@@ -1,6 +1,6 @@
 import { mkdir, writeFile } from 'fs/promises';
-import { delimiter, join } from 'path';
-import { getGlobalVariable } from '../utils/env';
+import { join } from 'path';
+import { getGlobalVariable, setGlobalVariable } from '../utils/env';
 
 /**
  * Configure npm to use a unique sandboxed environment.
@@ -8,23 +8,17 @@ import { getGlobalVariable } from '../utils/env';
 export default async function () {
   const tempRoot: string = getGlobalVariable('tmp-root');
   const npmModulesPrefix = join(tempRoot, 'npm-global');
+  const npmRegistry: string = getGlobalVariable('package-registry');
   const npmrc = join(tempRoot, '.npmrc');
 
   // Configure npm to use the sandboxed npm globals and rc file
+  // From this point onward all npm transactions use the "global" npm cache
+  // isolated within this e2e test invocation.
   process.env.NPM_CONFIG_USERCONFIG = npmrc;
   process.env.NPM_CONFIG_PREFIX = npmModulesPrefix;
 
-  // Ensure the custom npm global bin is first on the PATH
-  // https://docs.npmjs.com/cli/v8/configuring-npm/folders#executables
-  if (process.platform.startsWith('win')) {
-    process.env.PATH = npmModulesPrefix + delimiter + process.env.PATH;
-  } else {
-    process.env.PATH = join(npmModulesPrefix, 'bin') + delimiter + process.env.PATH;
-  }
-
-  // Ensure the globals directory and npmrc file exist.
-  // Configure the registry in the npmrc in addition to the environment variable.
-  await writeFile(npmrc, 'registry=' + getGlobalVariable('package-registry'));
+  // Configure the registry and prefix used within the test sandbox
+  await writeFile(npmrc, `registry=${npmRegistry}\nprefix=${npmModulesPrefix}`);
   await mkdir(npmModulesPrefix);
 
   console.log(`  Using "${npmModulesPrefix}" as e2e test global npm cache.`);

tests/legacy-cli/e2e/setup/100-global-cli.ts

@@ -1,5 +1,8 @@
 import { getGlobalVariable } from '../utils/env';
-import { exec, silentNpm } from '../utils/process';
+import { silentNpm } from '../utils/process';
+
+const NPM_VERSION = '7.24.0';
+const YARN_VERSION = '1.22.18';
 
 export default async function () {
   const argv = getGlobalVariable('argv');
@@ -9,10 +12,13 @@ export default async function () {
 
   const testRegistry: string = getGlobalVariable('package-registry');
 
-  // Install global Angular CLI.
-  await silentNpm('install', '--global', '@angular/cli', `--registry=${testRegistry}`);
-
-  try {
-    await exec(process.platform.startsWith('win') ? 'where' : 'which', 'ng');
-  } catch {}
+  // Install global Angular CLI being tested, npm+yarn used by e2e tests.
+  await silentNpm(
+    'install',
+    '--global',
+    `--registry=${testRegistry}`,
+    '@angular/cli',
+    `npm@${NPM_VERSION}`,
+    `yarn@${YARN_VERSION}`,
+  );
 }

tests/legacy-cli/e2e/tests/basic/e2e.ts

@@ -59,13 +59,6 @@ export default function () {
       // Should run side-by-side with `ng serve`
       .then(() => execAndWaitForOutputToMatch('ng', ['serve'], / Compiled successfully./))
       .then(() => ng('e2e', 'test-project', '--dev-server-target='))
-      // Should fail without updated webdriver
-      .then(() => replaceInFile('e2e/protractor.conf.js', /chromeDriver: String.raw`[^`]*`,/, ''))
-      .then(() =>
-        expectToFail(() =>
-          ng('e2e', 'test-project', '--no-webdriver-update', '--dev-server-target='),
-        ),
-      )
       .finally(() => killAllProcesses())
   );
 }

tests/legacy-cli/e2e/tests/packages/webpack/test-app.ts

@@ -3,7 +3,7 @@ import { createProjectFromAsset } from '../../../utils/assets';
 import { expectFileSizeToBeUnder, expectFileToMatch, replaceInFile } from '../../../utils/fs';
 import { execWithEnv } from '../../../utils/process';
 
-export default async function (skipCleaning: () => void) {
+export default async function () {
   const webpackCLIBin = normalize('node_modules/.bin/webpack-cli');
 
   await createProjectFromAsset('webpack/test-app');
@@ -30,6 +30,4 @@ export default async function (skipCleaning: () => void) {
     'DISABLE_V8_COMPILE_CACHE': '1',
   });
   await expectFileToMatch('dist/app.main.js', 'AppModule');
-
-  skipCleaning();
 }

tests/legacy-cli/e2e/utils/env.ts

@@ -1,12 +1,26 @@
-const global: { [name: string]: any } = Object.create(null);
+const ENV_PREFIX = 'LEGACY_CLI__';
 
 export function setGlobalVariable(name: string, value: any) {
-  global[name] = value;
+  if (value === undefined) {
+    delete process.env[ENV_PREFIX + name];
+  } else {
+    process.env[ENV_PREFIX + name] = JSON.stringify(value);
+  }
 }
 
 export function getGlobalVariable<T = any>(name: string): T {
-  if (!(name in global)) {
+  const value = process.env[ENV_PREFIX + name];
+  if (value === undefined) {
     throw new Error(`Trying to access variable "${name}" but it's not defined.`);
   }
-  return global[name] as T;
+  return JSON.parse(value) as T;
+}
+
+export function getGlobalVariablesEnv(): NodeJS.ProcessEnv {
+  return Object.keys(process.env)
+    .filter((v) => v.startsWith(ENV_PREFIX))
+    .reduce<NodeJS.ProcessEnv>((vars, n) => {
+      vars[n] = process.env[n];
+      return vars;
+    }, {});
 }

tests/legacy-cli/e2e/utils/process.ts

@@ -3,9 +3,10 @@ import { SpawnOptions } from 'child_process';
 import * as child_process from 'child_process';
 import { concat, defer, EMPTY, from } from 'rxjs';
 import { repeat, takeLast } from 'rxjs/operators';
-import { getGlobalVariable } from './env';
+import { getGlobalVariable, getGlobalVariablesEnv } from './env';
 import { catchError } from 'rxjs/operators';
 import treeKill from 'tree-kill';
+import { delimiter, join, resolve } from 'path';
 
 interface ExecOptions {
   silent?: boolean;
@@ -300,22 +301,21 @@ export function silentNpm(
       {
         silent: true,
         cwd: (options as { cwd?: string } | undefined)?.cwd,
-        env: extractNpmEnv(),
       },
       'npm',
       params,
     );
   } else {
-    return _exec({ silent: true, env: extractNpmEnv() }, 'npm', args as string[]);
+    return _exec({ silent: true }, 'npm', args as string[]);
   }
 }
 
 export function silentYarn(...args: string[]) {
-  return _exec({ silent: true, env: extractNpmEnv() }, 'yarn', args);
+  return _exec({ silent: true }, 'yarn', args);
 }
 
 export function npm(...args: string[]) {
-  return _exec({ env: extractNpmEnv() }, 'npm', args);
+  return _exec({}, 'npm', args);
 }
 
 export function node(...args: string[]) {
@@ -329,3 +329,39 @@ export function git(...args: string[]) {
 export function silentGit(...args: string[]) {
   return _exec({ silent: true }, 'git', args);
 }
+
+/**
+ * Launch the given entry in an child process isolated to the test environment.
+ *
+ * The test environment includes the local NPM registry, isolated NPM globals,
+ * the PATH variable only referencing the local node_modules and local NPM
+ * registry (not the test runner or standard global node_modules).
+ */
+export async function launchTestProcess(entry: string, ...args: any[]) {
+  const tempRoot: string = getGlobalVariable('tmp-root');
+
+  // Extract explicit environment variables for the test process.
+  const env: NodeJS.ProcessEnv = {
+    ...extractNpmEnv(),
+    ...getGlobalVariablesEnv(),
+  };
+
+  // Modify the PATH environment variable...
+  let paths = process.env.PATH!.split(delimiter);
+
+  // Only include paths within the sandboxed test environment or external
+  // non angular-cli paths such as /usr/bin for generic commands.
+  paths = paths.filter((p) => p.startsWith(tempRoot) || !p.includes('angular-cli'));
+
+  // Ensure the custom npm global bin is on the PATH
+  // https://docs.npmjs.com/cli/v8/configuring-npm/folders#executables
+  if (process.platform.startsWith('win')) {
+    paths.unshift(env.NPM_CONFIG_PREFIX!);
+  } else {
+    paths.unshift(join(env.NPM_CONFIG_PREFIX!, 'bin'));
+  }
+
+  env.PATH = paths.join(delimiter);
+
+  return _exec({ env }, process.execPath, [resolve(__dirname, 'run_test_process'), entry, ...args]);
+}

tests/legacy-cli/e2e/utils/project.ts

@@ -7,7 +7,7 @@ import { getGlobalVariable } from './env';
 import { prependToFile, readFile, replaceInFile, writeFile } from './fs';
 import { gitCommit } from './git';
 import { installWorkspacePackages } from './packages';
-import { execAndWaitForOutputToMatch, git, ng } from './process';
+import { exec, execAndWaitForOutputToMatch, git, ng } from './process';
 
 export function updateJsonFile(filePath: string, fn: (json: any) => any | void) {
   return readFile(filePath).then((tsConfigJson) => {
@@ -42,6 +42,26 @@ export async function prepareProjectForE2e(name: string) {
 
   await ng('generate', 'e2e', '--related-app-name', name);
 
+  // Initialize selenium webdriver.
+  // Often fails the first time so attempt twice if necessary.
+  const runWebdriverUpdate = () =>
+    exec(
+      'node',
+      'node_modules/protractor/bin/webdriver-manager',
+      'update',
+      '--standalone',
+      'false',
+      '--gecko',
+      'false',
+      '--versions.chrome',
+      '101.0.4951.41',
+    );
+  try {
+    await runWebdriverUpdate();
+  } catch (e) {
+    await runWebdriverUpdate();
+  }
+
   await useCIChrome('e2e');
   await useCIChrome('');
 

tests/legacy-cli/e2e/utils/run_test_process.js

@@ -0,0 +1,3 @@
+'use strict';
+require('../../../../lib/bootstrap-local');
+require('./test_process');

tests/legacy-cli/e2e/utils/test_process.ts

@@ -0,0 +1,19 @@
+import { killAllProcesses } from './process';
+
+const testScript: string = process.argv[2];
+const testModule = require(testScript);
+const testFunction: () => Promise<void> | void =
+  typeof testModule == 'function'
+    ? testModule
+    : typeof testModule.default == 'function'
+    ? testModule.default
+    : () => {
+        throw new Error('Invalid test module.');
+      };
+
+(async () => Promise.resolve(testFunction()))()
+  .finally(killAllProcesses)
+  .catch((e) => {
+    console.error(e);
+    process.exitCode = -1;
+  });

tests/legacy-cli/e2e_runner.ts

@@ -4,10 +4,12 @@ import * as colors from 'ansi-colors';
 import glob from 'glob';
 import yargsParser from 'yargs-parser';
 import * as path from 'path';
-import { setGlobalVariable } from './e2e/utils/env';
+import { getGlobalVariable, setGlobalVariable } from './e2e/utils/env';
 import { gitClean } from './e2e/utils/git';
 import { createNpmRegistry } from './e2e/utils/registry';
 import { AddressInfo, createServer, Server } from 'net';
+import { launchTestProcess } from './e2e/utils/process';
+import { join } from 'path';
 
 Error.stackTraceLimit = Infinity;
 
@@ -73,6 +75,7 @@ const testGlob = argv.glob || 'tests/**/*.ts';
 
 const e2eRoot = path.join(__dirname, 'e2e');
 const allSetups = glob.sync('setup/**/*.ts', { nodir: true, cwd: e2eRoot }).sort();
+const allInitializers = glob.sync('initialize/**/*.ts', { nodir: true, cwd: e2eRoot }).sort();
 const allTests = glob
   .sync(testGlob, { nodir: true, cwd: e2eRoot, ignore: argv.ignore })
   // Replace windows slashes.
@@ -133,6 +136,7 @@ Promise.all([findFreePort(), findFreePort()])
 
     try {
       await runSteps(runSetup, allSetups, 'setup');
+      await runSteps(runInitializer, allInitializers, 'initializer');
       await runSteps(runTest, testsToRun, 'test');
 
       console.log(colors.green('Done.'));
@@ -166,7 +170,7 @@ Promise.all([findFreePort(), findFreePort()])
 async function runSteps(
   run: (name: string) => Promise<void> | void,
   steps: string[],
-  type: 'setup' | 'test',
+  type: 'setup' | 'test' | 'initializer',
 ) {
   for (const [stepIndex, relativeName] of steps.entries()) {
     // Make sure this is a windows compatible path.
@@ -199,48 +203,37 @@ async function runSetup(absoluteName: string) {
   await (typeof module === 'function' ? module : module.default)();
 }
 
+/**
+ * Run a file from the projects root directory in a subprocess via launchTestProcess().
+ */
+async function runInitializer(absoluteName: string) {
+  process.chdir(getGlobalVariable('projects-root'));
+
+  await launchTestProcess(absoluteName);
+}
+
+/**
+ * Run a file from the main 'test-project' directory in a subprocess via launchTestProcess().
+ */
 async function runTest(absoluteName: string) {
-  const module = require(absoluteName);
-  const originalEnvVariables = {
-    ...process.env,
-  };
+  process.chdir(join(getGlobalVariable('projects-root'), 'test-project'));
 
-  const fn: (skipClean?: () => void) => Promise<void> | void =
-    typeof module == 'function'
-      ? module
-      : typeof module.default == 'function'
-      ? module.default
-      : () => {
-          throw new Error('Invalid test module.');
-        };
+  await launchTestProcess(absoluteName);
 
-  let clean = true;
-  let previousDir = process.cwd();
-
-  await fn(() => (clean = false));
-
-  // Change the directory back to where it was before the test.
-  // This allows tests to chdir without worrying about keeping the original directory.
-  if (previousDir) {
-    process.chdir(previousDir);
-
-    // Restore env variables before each test.
-    console.log('Restoring original environment variables...');
-    process.env = originalEnvVariables;
-  }
-
-  // Skip cleaning if the test requested an exception.
-  if (clean) {
-    logStack.push(new logging.NullLogger());
-    try {
-      await gitClean();
-    } finally {
-      logStack.pop();
-    }
+  logStack.push(new logging.NullLogger());
+  try {
+    await gitClean();
+  } finally {
+    logStack.pop();
   }
 }
 
-function printHeader(testName: string, testIndex: number, count: number, type: 'setup' | 'test') {
+function printHeader(
+  testName: string,
+  testIndex: number,
+  count: number,
+  type: 'setup' | 'initializer' | 'test',
+) {
   const text = `${testIndex + 1} of ${count}`;
   const fullIndex = testIndex * nbShards + shardId + 1;
   const shard =
@@ -254,7 +247,7 @@ function printHeader(testName: string, testIndex: number, count: number, type: '
   );
 }
 
-function printFooter(testName: string, type: 'setup' | 'test', startTime: number) {
+function printFooter(testName: string, type: 'setup' | 'initializer' | 'test', startTime: number) {
   // Round to hundredth of a second.
   const t = Math.round((Date.now() - startTime) / 10) / 100;
   console.log(colors.green(`Last ${type} took `) + colors.bold.blue('' + t) + colors.green('s...'));