Revert "cleanup(core): copy from cache only when needed"

This reverts commit 4dac0a2.

Author: FrozenPandaz

e2e/workspace/src/workspace.test.ts

@@ -15,7 +15,6 @@ import {
   updateFile,
   workspaceConfigName,
 } from '@nrwl/e2e/utils';
-import { TaskCacheStatus } from '@nrwl/workspace/src/utilities/output';
 
 describe('run-one', () => {
   let proj: string;
@@ -638,7 +637,7 @@ describe('cache', () => {
     });
     const outputWithBuildApp2Cached = runCLI(`affected:build ${files}`);
     expect(outputWithBuildApp2Cached).toContain('read the output from cache');
-    expectMatchedOutput(outputWithBuildApp2Cached, [myapp2]);
+    expectCached(outputWithBuildApp2Cached, [myapp2]);
 
     // touch package.json
     // --------------------------------------------
@@ -652,17 +651,13 @@ describe('cache', () => {
 
     // build individual project with caching
     const individualBuildWithCache = runCLI(`build ${myapp1}`);
-    expect(individualBuildWithCache).toContain(
-      TaskCacheStatus.MatchedExistingOutput
-    );
+    expect(individualBuildWithCache).toContain('from cache');
 
     // skip caching when building individual projects
     const individualBuildWithSkippedCache = runCLI(
       `build ${myapp1} --skip-nx-cache`
     );
-    expect(individualBuildWithSkippedCache).not.toContain(
-      TaskCacheStatus.MatchedExistingOutput
-    );
+    expect(individualBuildWithSkippedCache).not.toContain('from cache');
 
     // run lint with caching
     // --------------------------------------------
@@ -673,7 +668,7 @@ describe('cache', () => {
     expect(outputWithBothLintTasksCached).toContain(
       'read the output from cache'
     );
-    expectMatchedOutput(outputWithBothLintTasksCached, [
+    expectCached(outputWithBothLintTasksCached, [
       myapp1,
       myapp2,
       `${myapp1}-e2e`,
@@ -752,38 +747,19 @@ describe('cache', () => {
     actualOutput: string,
     expectedCachedProjects: string[]
   ) {
-    expectProjectMatchTaskCacheStatus(actualOutput, expectedCachedProjects);
-  }
-
-  function expectMatchedOutput(
-    actualOutput: string,
-    expectedMatchedOutputProjects: string[]
-  ) {
-    expectProjectMatchTaskCacheStatus(
-      actualOutput,
-      expectedMatchedOutputProjects,
-      TaskCacheStatus.MatchedExistingOutput
-    );
-  }
-
-  function expectProjectMatchTaskCacheStatus(
-    actualOutput: string,
-    expectedProjects: string[],
-    cacheStatus: TaskCacheStatus = TaskCacheStatus.RetrievedFromCache
-  ) {
-    const matchingProjects = [];
+    const cachedProjects = [];
     const lines = actualOutput.split('\n');
-    lines.forEach((s) => {
+    lines.forEach((s, i) => {
       if (s.startsWith(`> nx run`)) {
         const projectName = s.split(`> nx run `)[1].split(':')[0].trim();
-        if (s.indexOf(cacheStatus) > -1) {
-          matchingProjects.push(projectName);
+        if (s.indexOf('from cache') > -1) {
+          cachedProjects.push(projectName);
         }
       }
     });
 
-    matchingProjects.sort((a, b) => a.localeCompare(b));
-    expectedProjects.sort((a, b) => a.localeCompare(b));
-    expect(matchingProjects).toEqual(expectedProjects);
+    cachedProjects.sort((a, b) => a.localeCompare(b));
+    expectedCachedProjects.sort((a, b) => a.localeCompare(b));
+    expect(cachedProjects).toEqual(expectedCachedProjects);
   }
 });

packages/workspace/src/tasks-runner/cache.ts

@@ -12,7 +12,6 @@ import * as fsExtra from 'fs-extra';
 import { DefaultTasksRunnerOptions } from './default-tasks-runner';
 import { spawn } from 'child_process';
 import { cacheDirectory } from '../utilities/cache-directory';
-import { readJsonFile, writeJsonFile } from '../utilities/fileutils';
 
 export type CachedResult = { terminalOutput: string; outputsPath: string };
 export type TaskWithCachedResult = { task: Task; cachedResult: CachedResult };
@@ -39,7 +38,6 @@ export class Cache {
   root = appRootPath;
   cachePath = this.createCacheDir();
   terminalOutputsDir = this.createTerminalOutputsDir();
-  nxOutputsPath = this.ensureNxOutputsFile();
   cacheConfig = new CacheConfig(this.options);
 
   constructor(private readonly options: DefaultTasksRunnerOptions) {}
@@ -150,42 +148,6 @@ export class Cache {
     }
   }
 
-  removeOutputHashesFromNxOutputs(outputs: string[]): void {
-    if (outputs.length === 0) {
-      return;
-    }
-
-    const nxOutputs = readJsonFile(this.nxOutputsPath);
-    outputs.forEach((output) => {
-      delete nxOutputs[output];
-    });
-    writeJsonFile(this.nxOutputsPath, nxOutputs);
-  }
-
-  writeOutputHashesToNxOutputs(outputs: string[], hash: string): void {
-    if (outputs.length === 0) {
-      return;
-    }
-
-    const nxOutputs = readJsonFile(this.nxOutputsPath);
-    outputs.forEach((output) => {
-      nxOutputs[output] = hash;
-    });
-    writeJsonFile(this.nxOutputsPath, nxOutputs);
-  }
-
-  outputsMatchTask(task: Task, outputs: string[]): boolean {
-    if (outputs.length === 0) {
-      return true;
-    }
-
-    const nxOutputs = readJsonFile(this.nxOutputsPath);
-    return outputs.every(
-      (output) =>
-        existsSync(join(this.root, output)) && task.hash === nxOutputs[output]
-    );
-  }
-
   private getFromLocalDir(task: Task) {
     const tdCommit = join(this.cachePath, `${task.hash}.commit`);
     const td = join(this.cachePath, task.hash);
@@ -213,12 +175,4 @@ export class Cache {
     fsExtra.ensureDirSync(path);
     return path;
   }
-
-  private ensureNxOutputsFile() {
-    const path = join(this.cachePath, 'nx-outputs.json');
-    if (!existsSync(path)) {
-      writeJsonFile(path, {});
-    }
-    return path;
-  }
 }

packages/workspace/src/tasks-runner/task-orchestrator.ts

@@ -4,7 +4,7 @@ import * as dotenv from 'dotenv';
 import * as fs from 'fs';
 import { ProjectGraph } from '../core/project-graph';
 import { appRootPath } from '../utilities/app-root';
-import { output, TaskCacheStatus } from '../utilities/output';
+import { output } from '../utilities/output';
 import { Cache, TaskWithCachedResult } from './cache';
 import { DefaultTasksRunnerOptions } from './default-tasks-runner';
 import { AffectedEventType, Task } from './tasks-runner';
@@ -115,28 +115,18 @@ export class TaskOrchestrator {
     tasks.forEach((t) => {
       this.options.lifeCycle.startTask(t.task);
 
-      const outputs = getOutputs(this.projectGraph.nodes, t.task);
-      const outputsMatchCache = this.cache.outputsMatchTask(t.task, outputs);
-      if (!outputsMatchCache) {
-        this.cache.removeOutputHashesFromNxOutputs(outputs);
-        this.cache.copyFilesFromCache(t.cachedResult, outputs);
-        this.cache.writeOutputHashesToNxOutputs(outputs, t.task.hash);
-      }
-
       if (
         !this.initiatingProject ||
         this.initiatingProject === t.task.target.project
       ) {
         const args = this.getCommandArgs(t.task);
-        output.logCommand(
-          `nx ${args.join(' ')}`,
-          outputsMatchCache
-            ? TaskCacheStatus.MatchedExistingOutput
-            : TaskCacheStatus.RetrievedFromCache
-        );
+        output.logCommand(`nx ${args.join(' ')}`, true);
         process.stdout.write(t.cachedResult.terminalOutput);
       }
 
+      const outputs = getOutputs(this.projectGraph.nodes, t.task);
+      this.cache.copyFilesFromCache(t.cachedResult, outputs);
+
       this.options.lifeCycle.endTask(t.task, 0);
     });
 
@@ -185,7 +175,6 @@ export class TaskOrchestrator {
         if (forwardOutput) {
           output.logCommand(commandLine);
         }
-        this.cache.removeOutputHashesFromNxOutputs(taskOutputs);
         const p = fork(this.getCommand(), args, {
           stdio: ['inherit', 'pipe', 'pipe', 'ipc'],
           env,
@@ -221,23 +210,17 @@ export class TaskOrchestrator {
               this.cache
                 .put(task, outputPath, taskOutputs)
                 .then(() => {
-                  this.cache.writeOutputHashesToNxOutputs(
-                    taskOutputs,
-                    task.hash
-                  );
                   this.options.lifeCycle.endTask(task, code);
                   res(code);
                 })
                 .catch((e) => {
                   rej(e);
                 });
             } else {
-              this.cache.writeOutputHashesToNxOutputs(taskOutputs, task.hash);
               this.options.lifeCycle.endTask(task, code);
               res(code);
             }
           } else {
-            this.cache.writeOutputHashesToNxOutputs(taskOutputs, task.hash);
             this.options.lifeCycle.endTask(task, code);
             res(code);
           }
@@ -268,7 +251,6 @@ export class TaskOrchestrator {
         if (forwardOutput) {
           output.logCommand(commandLine);
         }
-        this.cache.removeOutputHashesFromNxOutputs(taskOutputs);
         const p = fork(this.getCommand(), args, {
           stdio: ['inherit', 'inherit', 'inherit', 'ipc'],
           env,
@@ -293,15 +275,13 @@ export class TaskOrchestrator {
             this.cache
               .put(task, outputPath, taskOutputs)
               .then(() => {
-                this.cache.writeOutputHashesToNxOutputs(taskOutputs, task.hash);
                 this.options.lifeCycle.endTask(task, code);
                 res(code);
               })
               .catch((e) => {
                 rej(e);
               });
           } else {
-            this.cache.writeOutputHashesToNxOutputs(taskOutputs, task.hash);
             this.options.lifeCycle.endTask(task, code);
             res(code);
           }

packages/workspace/src/utilities/output.ts

@@ -22,12 +22,6 @@ export interface CLISuccessMessageConfig {
   bodyLines?: string[];
 }
 
-export enum TaskCacheStatus {
-  NoCache = '[no cache]',
-  MatchedExistingOutput = '[existing outputs match the cache, left as is]',
-  RetrievedFromCache = '[retrieved from cache]',
-}
-
 /**
  * Automatically disable styling applied by chalk if CI=true
  */
@@ -183,16 +177,13 @@ class CLIOutput {
     this.addNewline();
   }
 
-  logCommand(
-    message: string,
-    cacheStatus: TaskCacheStatus = TaskCacheStatus.NoCache
-  ) {
+  logCommand(message: string, isCached: boolean = false) {
     this.addNewline();
 
     this.writeToStdOut(chalk.bold(`> ${message} `));
 
-    if (cacheStatus !== TaskCacheStatus.NoCache) {
-      this.writeToStdOut(chalk.bold.grey(cacheStatus));
+    if (isCached) {
+      this.writeToStdOut(chalk.bold.grey(`[retrieved from cache]`));
     }
 
     this.addNewline();