diff --git a/packages/ngtools/webpack/src/ivy/cache.ts b/packages/ngtools/webpack/src/ivy/cache.ts
index 1b3856b238..b6363a0351 100644
--- a/packages/ngtools/webpack/src/ivy/cache.ts
+++ b/packages/ngtools/webpack/src/ivy/cache.ts
@@ -11,36 +11,12 @@ import { normalizePath } from './paths';
 export class SourceFileCache extends Map<string, ts.SourceFile> {
   private readonly angularDiagnostics = new Map<ts.SourceFile, ts.Diagnostic[]>();
 
-  invalidate(
-    fileTimestamps: Map<string, 'ignore' | number | { safeTime: number } | null>,
-    buildTimestamp: number,
-  ): Set<string> {
-    const changedFiles = new Set<string>();
-    for (const [file, timeOrEntry] of fileTimestamps) {
-      if (timeOrEntry === 'ignore') {
-        continue;
-      }
-
-      let time;
-      if (typeof timeOrEntry === 'number') {
-        time = timeOrEntry;
-      } else if (timeOrEntry) {
-        time = timeOrEntry.safeTime;
-      }
-
-      if (!time || time >= buildTimestamp) {
-        // Cache stores paths using the POSIX directory separator
-        const normalizedFile = normalizePath(file);
-        const sourceFile = this.get(normalizedFile);
-        if (sourceFile) {
-          this.delete(normalizedFile);
-          this.angularDiagnostics.delete(sourceFile);
-        }
-        changedFiles.add(normalizedFile);
-      }
+  invalidate(file: string): void {
+    const sourceFile = this.get(file);
+    if (sourceFile) {
+      this.delete(file);
+      this.angularDiagnostics.delete(sourceFile);
     }
-
-    return changedFiles;
   }
 
   updateAngularDiagnostics(sourceFile: ts.SourceFile, diagnostics: ts.Diagnostic[]): void {
diff --git a/packages/ngtools/webpack/src/ivy/plugin.ts b/packages/ngtools/webpack/src/ivy/plugin.ts
index b35ea5edda..970d826b2e 100644
--- a/packages/ngtools/webpack/src/ivy/plugin.ts
+++ b/packages/ngtools/webpack/src/ivy/plugin.ts
@@ -95,7 +95,6 @@ export class AngularWebpackPlugin {
   private ngtscNextProgram?: NgtscProgram;
   private builder?: ts.EmitAndSemanticDiagnosticsBuilderProgram;
   private sourceFileCache?: SourceFileCache;
-  private buildTimestamp!: number;
   private readonly fileDependencies = new Map<string, Set<string>>();
   private readonly requiredFilesToEmit = new Set<string>();
   private readonly requiredFilesToEmitCache = new Map<string, EmitFileResult | undefined>();
@@ -204,12 +203,15 @@ export class AngularWebpackPlugin {
       let cache = this.sourceFileCache;
       let changedFiles;
       if (cache) {
-        // Invalidate existing cache based on compiler file timestamps
-        changedFiles = cache.invalidate(compiler.fileTimestamps, this.buildTimestamp);
+        changedFiles = new Set<string>();
+        for (const changedFile of [...compiler.modifiedFiles, ...compiler.removedFiles]) {
+          const normalizedChangedFile = normalizePath(changedFile);
+          // Invalidate file dependencies
+          this.fileDependencies.delete(normalizedChangedFile);
+          // Invalidate existing cache
+          cache.invalidate(normalizedChangedFile);
 
-        // Invalidate file dependencies of changed files
-        for (const changedFile of changedFiles) {
-          this.fileDependencies.delete(normalizePath(changedFile));
+          changedFiles.add(normalizedChangedFile);
         }
       } else {
         // Initialize a new cache
@@ -219,7 +221,6 @@ export class AngularWebpackPlugin {
           this.sourceFileCache = cache;
         }
       }
-      this.buildTimestamp = Date.now();
       augmentHostWithCaching(host, cache);
 
       const moduleResolutionCache = ts.createModuleResolutionCache(