refactor(@ngtools/webpack): avoid double caching of files

This commit is contained in:
Charles Lyding 2018-08-29 13:41:10 -04:00 committed by Keen Yee Liau
parent d29701f978
commit b6ff520b10
4 changed files with 160 additions and 307 deletions

View File

@ -314,6 +314,10 @@ export class AngularCompilerPlugin {
}
private _getTsProgram() {
if (!this._program) {
return undefined;
}
return this._JitMode ? this._program as ts.Program : (this._program as Program).getTsProgram();
}
@ -360,6 +364,12 @@ export class AngularCompilerPlugin {
// Use an identity function as all our paths are absolute already.
this._moduleResolutionCache = ts.createModuleResolutionCache(this._basePath, x => x);
const tsProgram = this._getTsProgram();
const oldFiles = new Set(tsProgram ?
tsProgram.getSourceFiles().map(sf => sf.fileName)
: [],
);
if (this._JitMode) {
// Create the TypeScript program.
time('AngularCompilerPlugin._createOrUpdateProgram.ts.createProgram');
@ -367,10 +377,15 @@ export class AngularCompilerPlugin {
this._rootNames,
this._compilerOptions,
this._compilerHost,
this._program as ts.Program,
tsProgram,
);
timeEnd('AngularCompilerPlugin._createOrUpdateProgram.ts.createProgram');
const newFiles = this._program.getSourceFiles().filter(sf => !oldFiles.has(sf.fileName));
for (const newFile of newFiles) {
this._compilerHost.invalidate(newFile.fileName);
}
return Promise.resolve();
} else {
time('AngularCompilerPlugin._createOrUpdateProgram.ng.createProgram');
@ -388,6 +403,12 @@ export class AngularCompilerPlugin {
return this._program.loadNgStructureAsync()
.then(() => {
timeEnd('AngularCompilerPlugin._createOrUpdateProgram.ng.loadNgStructureAsync');
const newFiles = (this._program as Program).getTsProgram()
.getSourceFiles().filter(sf => !oldFiles.has(sf.fileName));
for (const newFile of newFiles) {
this._compilerHost.invalidate(newFile.fileName);
}
});
}
})
@ -396,7 +417,7 @@ export class AngularCompilerPlugin {
if (!this._entryModule && this._mainPath) {
time('AngularCompilerPlugin._make.resolveEntryModuleFromMain');
this._entryModule = resolveEntryModuleFromMain(
this._mainPath, this._compilerHost, this._getTsProgram());
this._mainPath, this._compilerHost, this._getTsProgram() as ts.Program);
timeEnd('AngularCompilerPlugin._make.resolveEntryModuleFromMain');
}
});
@ -621,7 +642,6 @@ export class AngularCompilerPlugin {
this._basePath,
host,
);
webpackCompilerHost.enableCaching();
// Create and set a new WebpackResourceLoader.
this._resourceLoader = new WebpackResourceLoader();
@ -811,7 +831,7 @@ export class AngularCompilerPlugin {
? { path: workaroundResolve(this.entryModule.path), className: this.entryModule.className }
: this.entryModule;
const getLazyRoutes = () => this._lazyRoutes;
const getTypeChecker = () => this._getTsProgram().getTypeChecker();
const getTypeChecker = () => (this._getTsProgram() as ts.Program).getTypeChecker();
if (this._JitMode) {
// Replace resources in JIT.
@ -868,13 +888,15 @@ export class AngularCompilerPlugin {
// We need to run the `listLazyRoutes` the first time because it also navigates libraries
// and other things that we might miss using the (faster) findLazyRoutesInAst.
// Lazy routes modules will be read with compilerHost and added to the changed files.
const changedTsFiles = this._getChangedTsFiles();
if (this._ngCompilerSupportsNewApi) {
this._processLazyRoutes(this._listLazyRoutesFromProgram());
} else if (this._firstRun) {
this._processLazyRoutes(this._getLazyRoutesFromNgtools());
} else if (changedTsFiles.length > 0) {
this._processLazyRoutes(this._findLazyRoutesInAst(changedTsFiles));
} else {
const changedTsFiles = this._getChangedTsFiles();
if (changedTsFiles.length > 0) {
this._processLazyRoutes(this._findLazyRoutesInAst(changedTsFiles));
}
}
if (this._options.additionalLazyModules) {
this._processLazyRoutes(this._options.additionalLazyModules);
@ -887,7 +909,7 @@ export class AngularCompilerPlugin {
// We now have the final list of changed TS files.
// Go through each changed file and add transforms as needed.
const sourceFiles = this._getChangedTsFiles()
.map((fileName) => this._getTsProgram().getSourceFile(fileName))
.map((fileName) => (this._getTsProgram() as ts.Program).getSourceFile(fileName))
// At this point we shouldn't need to filter out undefined files, because any ts file
// that changed should be emitted.
// But due to hostReplacementPaths there can be files (the environment files)
@ -973,7 +995,7 @@ export class AngularCompilerPlugin {
} else {
// Check if the TS input file and the JS output file exist.
if (((fileName.endsWith('.ts') || fileName.endsWith('.tsx'))
&& !this._compilerHost.fileExists(fileName, false))
&& !this._compilerHost.fileExists(fileName))
|| !this._compilerHost.fileExists(outputFile, false)) {
let msg = `${fileName} is missing from the TypeScript compilation. `
+ `Please make sure it is in your tsconfig via the 'files' or 'include' property.`;
@ -1066,15 +1088,26 @@ export class AngularCompilerPlugin {
}
if (!hasErrors(allDiagnostics)) {
sourceFiles.forEach((sf) => {
const timeLabel = `AngularCompilerPlugin._emit.ts+${sf.fileName}+.emit`;
time(timeLabel);
emitResult = tsProgram.emit(sf, undefined, undefined, undefined,
if (this._firstRun || sourceFiles.length > 20) {
emitResult = tsProgram.emit(
undefined,
undefined,
undefined,
undefined,
{ before: this._transformers },
);
allDiagnostics.push(...emitResult.diagnostics);
timeEnd(timeLabel);
});
} else {
sourceFiles.forEach((sf) => {
const timeLabel = `AngularCompilerPlugin._emit.ts+${sf.fileName}+.emit`;
time(timeLabel);
emitResult = tsProgram.emit(sf, undefined, undefined, undefined,
{ before: this._transformers },
);
allDiagnostics.push(...emitResult.diagnostics);
timeEnd(timeLabel);
});
}
}
} else {
const angularProgram = program as Program;

View File

@ -5,10 +5,15 @@
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import { Path, getSystemPath, join as _join, normalize, virtualFs } from '@angular-devkit/core';
import { NodeJsSyncHost } from '@angular-devkit/core/node';
import * as fs from 'fs';
import { basename, dirname, join } from 'path';
import {
Path,
getSystemPath,
isAbsolute,
join,
normalize,
virtualFs,
} from '@angular-devkit/core';
import { Stats } from 'fs';
import * as ts from 'typescript';
import { WebpackResourceLoader } from './resource_loader';
@ -21,157 +26,26 @@ export interface OnErrorFn {
const dev = Math.floor(Math.random() * 10000);
export class VirtualStats implements fs.Stats {
protected _ctime = new Date();
protected _mtime = new Date();
protected _atime = new Date();
protected _btime = new Date();
protected _dev = dev;
protected _ino = Math.floor(Math.random() * 100000);
protected _mode = parseInt('777', 8); // RWX for everyone.
protected _uid = Number(process.env['UID']) || 0;
protected _gid = Number(process.env['GID']) || 0;
constructor(protected _path: string) {}
isFile() { return false; }
isDirectory() { return false; }
isBlockDevice() { return false; }
isCharacterDevice() { return false; }
isSymbolicLink() { return false; }
isFIFO() { return false; }
isSocket() { return false; }
get dev() { return this._dev; }
get ino() { return this._ino; }
get mode() { return this._mode; }
get nlink() { return 1; } // Default to 1 hard link.
get uid() { return this._uid; }
get gid() { return this._gid; }
get rdev() { return 0; }
get size() { return 0; }
get blksize() { return 512; }
get blocks() { return Math.ceil(this.size / this.blksize); }
get atime() { return this._atime; }
get atimeMs() { return this._atime.getTime(); }
get mtime() { return this._mtime; }
get mtimeMs() { return this._mtime.getTime(); }
get ctime() { return this._ctime; }
get ctimeMs() { return this._ctime.getTime(); }
get birthtime() { return this._btime; }
get birthtimeMs() { return this._btime.getTime(); }
}
export class VirtualDirStats extends VirtualStats {
constructor(_fileName: string) {
super(_fileName);
}
isDirectory() { return true; }
get size() { return 1024; }
}
export class VirtualFileStats extends VirtualStats {
private _sourceFile: ts.SourceFile | null;
private _content: string | null;
private _bufferContent: virtualFs.FileBuffer | null;
constructor(_fileName: string) {
super(_fileName);
}
static createFromString(_fileName: string, _content: string) {
const stats = new VirtualFileStats(_fileName);
stats.content = _content;
return stats;
}
static createFromBuffer(_fileName: string, _buffer: virtualFs.FileBuffer) {
const stats = new VirtualFileStats(_fileName);
stats.bufferContent = _buffer;
return stats;
}
get content() {
if (!this._content && this.bufferContent) {
this._content = virtualFs.fileBufferToString(this.bufferContent);
}
return this._content || '';
}
set content(v: string) {
this._content = v;
this._bufferContent = null;
this.resetMetadata();
}
get bufferContent() {
if (!this._bufferContent && this._content) {
this._bufferContent = virtualFs.stringToFileBuffer(this._content);
}
return this._bufferContent || virtualFs.stringToFileBuffer('');
}
set bufferContent(buf: virtualFs.FileBuffer) {
this._bufferContent = buf;
this._content = null;
this.resetMetadata();
}
setSourceFile(sourceFile: ts.SourceFile) {
this._sourceFile = sourceFile;
}
getSourceFile(languageVersion: ts.ScriptTarget, setParentNodes: boolean) {
if (!this._sourceFile) {
this._sourceFile = ts.createSourceFile(
workaroundResolve(this._path),
this.content,
languageVersion,
setParentNodes);
}
return this._sourceFile;
}
private resetMetadata(): void {
this._mtime = new Date();
this._sourceFile = null;
}
isFile() { return true; }
get size() { return this.content.length; }
}
export class WebpackCompilerHost implements ts.CompilerHost {
private _syncHost: virtualFs.SyncDelegateHost;
private _files: {[path: string]: VirtualFileStats | null} = Object.create(null);
private _directories: {[path: string]: VirtualDirStats | null} = Object.create(null);
private _changedFiles: {[path: string]: boolean} = Object.create(null);
private _changedDirs: {[path: string]: boolean} = Object.create(null);
private _basePath: string;
private _setParentNodes: boolean;
private _cache = false;
private _resourceLoader?: WebpackResourceLoader | undefined;
private _changedFiles = new Set<string>();
private _basePath: Path;
private _resourceLoader?: WebpackResourceLoader;
constructor(
private _options: ts.CompilerOptions,
basePath: string,
private _host: virtualFs.Host<fs.Stats> = new NodeJsSyncHost(),
host: virtualFs.Host,
) {
this._syncHost = new virtualFs.SyncDelegateHost(_host);
this._setParentNodes = true;
this._basePath = this._normalizePath(basePath);
this._syncHost = new virtualFs.SyncDelegateHost(new virtualFs.CordHost(host));
this._basePath = normalize(basePath);
}
private _normalizePath(path: string): Path {
return normalize(path);
private get virtualFiles(): Path[] {
return (this._syncHost.delegate as virtualFs.CordHost)
.records()
.filter(record => record.kind === 'create')
.map((record: virtualFs.CordHostCreate) => record.path);
}
denormalizePath(path: string) {
@ -179,167 +53,115 @@ export class WebpackCompilerHost implements ts.CompilerHost {
}
resolve(path: string): Path {
const p = this._normalizePath(path);
if (p[0] == '.') {
return this._normalizePath(join(this.getCurrentDirectory(), p));
} else if (p[0] == '/' || p.match(/^\w:\//)) {
const p = normalize(path);
if (isAbsolute(p)) {
return p;
} else {
return this._normalizePath(join(this._basePath, p));
return join(this._basePath, p);
}
}
private _cacheFile(fileName: string, stats: VirtualFileStats) {
this._files[fileName] = stats;
let p = dirname(fileName);
while (p && !this._directories[p]) {
this._directories[p] = new VirtualDirStats(p);
this._changedDirs[p] = true;
p = dirname(p);
}
this._changedFiles[fileName] = true;
}
get dirty() {
return Object.keys(this._changedFiles).length > 0;
}
enableCaching() {
this._cache = true;
}
resetChangedFileTracker() {
this._changedFiles = Object.create(null);
this._changedDirs = Object.create(null);
this._changedFiles.clear();
}
getChangedFilePaths(): string[] {
return Object.keys(this._changedFiles);
return [...this._changedFiles];
}
getNgFactoryPaths(): string[] {
return Object.keys(this._files)
return this.virtualFiles
.filter(fileName => fileName.endsWith('.ngfactory.js') || fileName.endsWith('.ngstyle.js'))
// These paths are used by the virtual file system decorator so we must denormalize them.
.map(path => this.denormalizePath(path as Path));
.map(path => this.denormalizePath(path));
}
invalidate(fileName: string): void {
const fullPath = this.resolve(fileName);
if (fullPath in this._files) {
this._files[fullPath] = null;
} else {
for (const file in this._files) {
if (file.startsWith(fullPath + '/')) {
this._files[file] = null;
}
}
}
if (this.fileExists(fullPath)) {
this._changedFiles[fullPath] = true;
if (this.fileExists(fileName)) {
this._changedFiles.add(fullPath);
}
}
fileExists(fileName: string, delegate = true): boolean {
const p = this.resolve(fileName);
return this._files[p] != null
|| (delegate && this._syncHost.exists(normalize(p)));
const exists = this._syncHost.exists(p) && this._syncHost.isFile(p);
if (delegate) {
return exists;
} else {
const backend = new virtualFs.SyncDelegateHost(
(this._syncHost.delegate as virtualFs.CordHost).backend as virtualFs.Host,
);
return exists && !(backend.exists(p) && backend.isFile(p));
}
}
readFile(fileName: string): string | undefined {
const stats = this.findVirtualFile(fileName);
const filePath = this.resolve(fileName);
if (!this._syncHost.exists(filePath) || !this._syncHost.isFile(filePath)) {
return undefined;
}
return stats && stats.content;
return virtualFs.fileBufferToString(this._syncHost.read(filePath));
}
readFileBuffer(fileName: string): Buffer | undefined {
const stats = this.findVirtualFile(fileName);
if (stats) {
const buffer = Buffer.from(stats.bufferContent);
return buffer;
}
}
private findVirtualFile(fileName: string): VirtualFileStats | undefined {
const p = this.resolve(fileName);
const stats = this._files[p];
if (stats) {
return stats;
}
try {
const fileBuffer = this._syncHost.read(p);
if (fileBuffer) {
const stats = VirtualFileStats.createFromBuffer(p, fileBuffer);
if (this._cache) {
this._cacheFile(p, stats);
}
return stats;
}
} catch {
const filePath = this.resolve(fileName);
if (!this._syncHost.exists(filePath) || !this._syncHost.isFile(filePath)) {
return undefined;
}
return Buffer.from(this._syncHost.read(filePath));
}
stat(path: string): VirtualStats | null {
stat(path: string): Stats | null {
const p = this.resolve(path);
const stats = this._files[p] || this._directories[p];
const stats = this._syncHost.exists(p) && this._syncHost.stat(p);
if (!stats) {
return this._syncHost.stat(p) as VirtualStats | null;
return null;
}
return stats;
return {
isBlockDevice: () => false,
isCharacterDevice: () => false,
isFIFO: () => false,
isSymbolicLink: () => false,
isSocket: () => false,
dev,
ino: Math.floor(Math.random() * 100000),
mode: parseInt('777', 8),
nlink: 1,
uid: 0,
gid: 0,
rdev: 0,
blksize: 512,
blocks: Math.ceil(stats.size / 512),
atimeMs: stats.atime.getTime(),
mtimeMs: stats.mtime.getTime(),
ctimeMs: stats.ctime.getTime(),
birthtimeMs: stats.birthtime.getTime(),
...stats,
};
}
directoryExists(directoryName: string, delegate = true): boolean {
directoryExists(directoryName: string): boolean {
const p = this.resolve(directoryName);
return (this._directories[p] != null)
|| (delegate && this._syncHost.exists(p) && this._syncHost.isDirectory(p));
}
getFiles(path: string): string[] {
const p = this.resolve(path);
const subfiles = Object.keys(this._files)
.filter(fileName => dirname(fileName) == p)
.map(p => basename(p));
let delegated: string[];
try {
delegated = this._syncHost.list(p).filter((x: string) => {
try {
return this._syncHost.isFile(_join(p, x));
} catch {
return false;
}
});
} catch {
delegated = [];
}
return delegated.concat(subfiles);
return this._syncHost.exists(p) && this._syncHost.isDirectory(p);
}
getDirectories(path: string): string[] {
const p = this.resolve(path);
const subdirs = Object.keys(this._directories)
.filter(fileName => dirname(fileName) == p)
.map(path => basename(path));
let delegated: string[];
try {
delegated = this._syncHost.list(p).filter((x: string) => {
delegated = this._syncHost.list(p).filter(x => {
try {
return this._syncHost.isDirectory(_join(p, x));
return this._syncHost.isDirectory(join(p, x));
} catch {
return false;
}
@ -348,45 +170,26 @@ export class WebpackCompilerHost implements ts.CompilerHost {
delegated = [];
}
return delegated.concat(subdirs);
return delegated;
}
getSourceFile(fileName: string, languageVersion: ts.ScriptTarget, _onError?: OnErrorFn) {
fileName = this.resolve(fileName);
let stats = this._files[fileName];
if (!stats) {
getSourceFile(fileName: string, languageVersion: ts.ScriptTarget, onError?: OnErrorFn) {
try {
const content = this.readFile(fileName);
if (!this._cache && content) {
return ts.createSourceFile(
workaroundResolve(fileName),
content,
languageVersion,
this._setParentNodes,
);
} else {
stats = this._files[fileName];
if (!stats) {
// If cache is turned on and the file exists, the readFile call will have populated stats.
// Empty stats at this point mean the file doesn't exist at and so we should return
// undefined.
return undefined;
}
if (content != undefined) {
return ts.createSourceFile(workaroundResolve(fileName), content, languageVersion, true);
}
} catch (e) {
if (onError) {
onError(e.message);
}
}
return stats && stats.getSourceFile(languageVersion, this._setParentNodes);
}
get getCancellationToken() {
// return this._delegate.getCancellationToken;
// TODO: consider implementing a cancellation token.
return undefined;
}
getDefaultLibFileName(options: ts.CompilerOptions) {
return ts.createCompilerHost(options, false).getDefaultLibFileName(options);
return ts.createCompilerHost(options).getDefaultLibFileName(options);
}
// This is due to typescript CompilerHost interface being weird on writeFile. This shuts down
@ -396,21 +199,29 @@ export class WebpackCompilerHost implements ts.CompilerHost {
fileName: string,
data: string,
_writeByteOrderMark: boolean,
_onError?: (message: string) => void,
onError?: (message: string) => void,
_sourceFiles?: ReadonlyArray<ts.SourceFile>,
): void => {
const p = this.resolve(fileName);
const stats = VirtualFileStats.createFromString(p, data);
this._cacheFile(p, stats);
try {
this._syncHost.write(p, virtualFs.stringToFileBuffer(data));
} catch (e) {
if (onError) {
onError(e.message);
}
}
};
}
getCurrentDirectory(): string {
return this._basePath !== null ? this._basePath : '/';
return this._basePath;
}
getCanonicalFileName(fileName: string): string {
return this.resolve(fileName);
const path = this.resolve(fileName);
return this.useCaseSensitiveFileNames ? path : path.toLowerCase();
}
useCaseSensitiveFileNames(): boolean {

View File

@ -5,6 +5,7 @@
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import { virtualFs } from '@angular-devkit/core';
import * as ts from 'typescript';
import { WebpackCompilerHost } from '../compiler_host';
@ -57,7 +58,11 @@ export function createTypescriptContext(content: string) {
};
// Create compiler host.
const compilerHost = new WebpackCompilerHost(compilerOptions, basePath);
const compilerHost = new WebpackCompilerHost(
compilerOptions,
basePath,
new virtualFs.SimpleMemoryHost(),
);
// Add a dummy file to host content.
compilerHost.writeFile(fileName, content, false);

View File

@ -6,6 +6,7 @@
* found in the LICENSE file at https://angular.io/license
*/
import { terminal } from '@angular-devkit/core';
import { NodeJsSyncHost } from '@angular-devkit/core/node';
import * as ts from 'typescript';
import { time, timeEnd } from './benchmark';
import { WebpackCompilerHost } from './compiler_host';
@ -62,8 +63,11 @@ export class TypeChecker {
private _rootNames: string[],
) {
time('TypeChecker.constructor');
const compilerHost = new WebpackCompilerHost(_compilerOptions, _basePath);
compilerHost.enableCaching();
const compilerHost = new WebpackCompilerHost(
_compilerOptions,
_basePath,
new NodeJsSyncHost(),
);
// We don't set a async resource loader on the compiler host because we only support
// html templates, which are the only ones that can throw errors, and those can be loaded
// synchronously.