refactor: speed up sourcemap lookups by checking existence of .map files (#1780)

* If .map file exists no need to parse it

* add line space

* code comment

* update to use previously read children

* fix root not getting touched when retrieved

* avoid extra stat in createMetadataForFile

* fixup tests

---------

Co-authored-by: Sunil Surana <[email protected]>
Co-authored-by: Connor Peet <[email protected]>

Author: 3 people

src/common/sourceMaps/cacheTree.ts

@@ -20,6 +20,7 @@ export namespace CacheTree {
    * separated with forward slashes.
    */
   export function getPath<T>(node: CacheTree<T>, directory: string) {
+    node[touched] = 1;
     return _getDir(node, splitDir(directory), 0);
   }
 

src/common/sourceMaps/sourceMapRepository.ts

@@ -2,11 +2,11 @@
  * Copyright (C) Microsoft Corporation. All rights reserved.
  *--------------------------------------------------------*/
 
-import { xxHash32 } from 'js-xxhash';
+import { basename } from 'path';
 import { FileGlobList } from '../fileGlobList';
-import { readfile, stat } from '../fsUtils';
+import { readfile } from '../fsUtils';
 import { parseSourceMappingUrl } from '../sourceUtils';
-import { absolutePathToFileUrl, completeUrl, fileUrlToAbsolutePath, isDataUri } from '../urlUtils';
+import { absolutePathToFileUrl, completeUrl, isDataUri } from '../urlUtils';
 import { ISourceMapMetadata } from './sourceMap';
 
 /**
@@ -67,13 +67,21 @@ export interface ISearchStrategy {
  */
 export const createMetadataForFile = async (
   compiledPath: string,
+  metadata: { siblings: readonly string[]; mtime: number },
   fileContents?: string,
 ): Promise<Required<ISourceMapMetadata> | undefined> => {
-  if (typeof fileContents === 'undefined') {
-    fileContents = await readfile(compiledPath);
+  let sourceMapUrl;
+  const compiledFileName = basename(compiledPath);
+  const maybeSibling = `${compiledFileName}.map`;
+  if (metadata.siblings.includes(maybeSibling)) {
+    sourceMapUrl = maybeSibling;
+  }
+  if (!sourceMapUrl) {
+    if (typeof fileContents === 'undefined') {
+      fileContents = await readfile(compiledPath);
+    }
+    sourceMapUrl = parseSourceMappingUrl(fileContents);
   }
-
-  let sourceMapUrl = parseSourceMappingUrl(fileContents);
   if (!sourceMapUrl) {
     return;
   }
@@ -91,20 +99,9 @@ export const createMetadataForFile = async (
     return;
   }
 
-  let cacheKey: number;
-  if (smIsDataUri) {
-    cacheKey = xxHash32(sourceMapUrl);
-  } else {
-    const stats = await stat(fileUrlToAbsolutePath(sourceMapUrl) || compiledPath);
-    if (!stats) {
-      return; // ENOENT, usually
-    }
-    cacheKey = stats.mtimeMs;
-  }
-
   return {
     compiledPath,
     sourceMapUrl,
-    cacheKey,
+    cacheKey: metadata.mtime,
   };
 };

src/common/sourceMaps/turboGlobStream.test.ts

@@ -31,7 +31,15 @@ describe('TurboGlobStream', () => {
     for (let i = 0; i < 2; i++) {
       await new Promise((resolve, reject) => {
         const matches: T[] = [];
-        const tgs = new TurboGlobStream({ cwd: dir, ...opts, cache });
+        const tgs = new TurboGlobStream({
+          cwd: dir,
+          ...opts,
+          cache,
+          fileProcessor: (fname, meta) => {
+            delete (meta as Record<string, unknown>).mtime; // delete this since it'll change for every test
+            return opts.fileProcessor(fname, meta);
+          },
+        });
         tgs.onError(reject);
         tgs.onFile(result => matches.push(result));
         tgs.done
@@ -80,7 +88,9 @@ describe('TurboGlobStream', () => {
       },
     });
 
-    expect(fileProcessor.callCount).to.equal(1);
+    expect(fileProcessor.args).to.deep.equal([
+      [join(dir, 'a', 'a1.js'), { siblings: ['a1.js', 'a2.js'] }],
+    ]);
   });
 
   it('uses platform preferred path', async () => {
@@ -109,7 +119,10 @@ describe('TurboGlobStream', () => {
       },
     });
 
-    expect(fileProcessor.callCount).to.equal(2);
+    expect(fileProcessor.args.slice().sort((a, b) => a[0].localeCompare(b[0]))).to.deep.equal([
+      [join(dir, 'a', 'a1.js'), { siblings: ['a1.js', 'a2.js'] }],
+      [join(dir, 'a', 'a2.js'), { siblings: ['a1.js', 'a2.js'] }],
+    ]);
   });
 
   it('globs for files recursively', async () => {
@@ -123,7 +136,13 @@ describe('TurboGlobStream', () => {
       },
     });
 
-    expect(fileProcessor.callCount).to.equal(5);
+    expect(fileProcessor.args.slice().sort((a, b) => a[0].localeCompare(b[0]))).to.deep.equal([
+      [join(dir, 'a', 'a1.js'), { siblings: ['a1.js', 'a2.js'] }],
+      [join(dir, 'a', 'a2.js'), { siblings: ['a1.js', 'a2.js'] }],
+      [join(dir, 'b', 'b1.js'), { siblings: ['b1.js', 'b2.js'] }],
+      [join(dir, 'b', 'b2.js'), { siblings: ['b1.js', 'b2.js'] }],
+      [join(dir, 'c', 'nested', 'a', 'c1.js'), { siblings: ['c1.js'] }],
+    ]);
   });
 
   it('globs star dirname', async () => {

src/common/sourceMaps/turboGlobStream.ts

@@ -29,6 +29,11 @@ interface ITokensContext {
   seen: Set<string>;
 }
 
+export type FileProcessorFn<T> = (
+  path: string,
+  metadata: { siblings: readonly string[]; mtime: number },
+) => Promise<T>;
+
 export interface ITurboGlobStreamOptions<E> {
   /** Glob patterns */
   pattern: string;
@@ -41,7 +46,7 @@ export interface ITurboGlobStreamOptions<E> {
   /** Cache state, will be updated. */
   cache: CacheTree<IGlobCached<E>>;
   /** File to transform a path into extracted data emitted on onFile */
-  fileProcessor: (path: string) => Promise<E>;
+  fileProcessor: FileProcessorFn<E>;
 }
 
 const forwardSlashRe = /\//g;
@@ -60,7 +65,7 @@ export class TurboGlobStream<E> {
 
   private readonly filter?: (path: string, previousData?: E) => boolean;
   private readonly ignore: ((path: string) => boolean)[];
-  private readonly processor: (path: string) => Promise<E>;
+  private readonly processor: FileProcessorFn<E>;
   private readonly fileEmitter = new EventEmitter<E>();
   public readonly onFile = this.fileEmitter.event;
   private readonly errorEmitter = new EventEmitter<{ path: string; error: Error }>();
@@ -121,7 +126,7 @@ export class TurboGlobStream<E> {
         const depths = tokens[0] === '**' ? [0, 1] : [0];
         const cacheEntry = CacheTree.getPath(opts.cache, opts.cwd);
         const ctx = { elements: tokens, seen: new Set<string>() };
-        return Promise.all(depths.map(d => this.readSomething(ctx, d, opts.cwd, cacheEntry)));
+        return Promise.all(depths.map(d => this.readSomething(ctx, d, opts.cwd, [], cacheEntry)));
       }),
     ).then(() => undefined);
   }
@@ -134,6 +139,7 @@ export class TurboGlobStream<E> {
     ctx: ITokensContext,
     ti: number,
     path: string,
+    siblings: readonly string[],
     cache: CacheTree<IGlobCached<E>>,
   ) {
     // Skip already processed files, since we might see them twice during glob stars.
@@ -168,19 +174,32 @@ export class TurboGlobStream<E> {
       // children added or removed.
       if (cd.type === CachedType.Directory) {
         const todo: unknown[] = [];
+        const entries = Object.entries(cache.children);
+        const siblings = entries
+          .filter(([, e]) => e.data?.type !== CachedType.Directory)
+          .map(([n]) => n);
+
         for (const [name, child] of Object.entries(cache.children)) {
           // for cached objects with a type, recurse normally. For ones without,
           // try to stat them first (may have been interrupted before they were finished)
           todo.push(
             child.data !== undefined
-              ? this.handleDirectoryEntry(ctx, ti, path, { name, type: child.data.type }, cache)
+              ? this.handleDirectoryEntry(
+                  ctx,
+                  ti,
+                  path,
+                  { name, type: child.data.type },
+                  siblings,
+                  cache,
+                )
               : this.stat(path).then(
                   stat =>
                     this.handleDirectoryEntry(
                       ctx,
                       ti,
                       path,
                       { name, type: stat.isFile() ? CachedType.File : CachedType.Directory },
+                      siblings,
                       cache,
                     ),
                   () => undefined,
@@ -200,7 +219,7 @@ export class TurboGlobStream<E> {
       await this.handleDir(ctx, ti, stat.mtimeMs, path, cache);
     } else {
       this.alreadyProcessedFiles.add(cache);
-      await this.handleFile(stat.mtimeMs, path, cache);
+      await this.handleFile(stat.mtimeMs, path, siblings, cache);
     }
   }
 
@@ -230,6 +249,7 @@ export class TurboGlobStream<E> {
     ti: number,
     path: string,
     dirent: { name: string; type: CachedType },
+    siblings: readonly string[],
     cache: CacheTree<IGlobCached<E>>,
   ): unknown {
     const nextPath = path + '/' + dirent.name;
@@ -246,9 +266,11 @@ export class TurboGlobStream<E> {
     }
 
     if (typeof descends === 'number') {
-      return this.readSomething(ctx, ti + descends, nextPath, nextChild);
+      return this.readSomething(ctx, ti + descends, nextPath, siblings, nextChild);
     } else {
-      return Promise.all(descends.map(d => this.readSomething(ctx, ti + d, nextPath, nextChild)));
+      return Promise.all(
+        descends.map(d => this.readSomething(ctx, ti + d, nextPath, siblings, nextChild)),
+      );
     }
   }
 
@@ -295,11 +317,16 @@ export class TurboGlobStream<E> {
     }
   }
 
-  private async handleFile(mtime: number, path: string, cache: CacheTree<IGlobCached<E>>) {
+  private async handleFile(
+    mtime: number,
+    path: string,
+    siblings: readonly string[],
+    cache: CacheTree<IGlobCached<E>>,
+  ) {
     const platformPath = sep === '/' ? path : path.replace(forwardSlashRe, sep);
     let extracted: E;
     try {
-      extracted = await this.processor(platformPath);
+      extracted = await this.processor(platformPath, { siblings, mtime });
     } catch (error) {
       this.errorEmitter.fire({ path: platformPath, error });
       return;
@@ -325,6 +352,7 @@ export class TurboGlobStream<E> {
     }
 
     const todo: unknown[] = [];
+    const siblings = files.filter(f => f.isFile()).map(f => f.name);
     for (const file of files) {
       if (file.name.startsWith('.')) {
         continue;
@@ -339,7 +367,9 @@ export class TurboGlobStream<E> {
         continue;
       }
 
-      todo.push(this.handleDirectoryEntry(ctx, ti, path, { name: file.name, type }, cache));
+      todo.push(
+        this.handleDirectoryEntry(ctx, ti, path, { name: file.name, type }, siblings, cache),
+      );
     }
 
     await Promise.all(todo);

src/common/sourceMaps/turboSearchStrategy.ts

@@ -10,9 +10,9 @@ import { truthy } from '../objUtils';
 import { fixDriveLetterAndSlashes } from '../pathUtils';
 import { CacheTree } from './cacheTree';
 import {
-  createMetadataForFile,
   ISearchStrategy,
   ISourcemapStreamOptions,
+  createMetadataForFile,
 } from './sourceMapRepository';
 import { IGlobCached, TurboGlobStream } from './turboGlobStream';
 
@@ -82,7 +82,8 @@ export class TurboSearchStrategy implements ISearchStrategy {
       cwd: glob.cwd,
       cache,
       filter: opts.filter,
-      fileProcessor: file => createMetadataForFile(file).then(m => m && opts.processMap(m)),
+      fileProcessor: (file, metadata) =>
+        createMetadataForFile(file, metadata).then(m => m && opts.processMap(m)),
     });
 
     tgs.onError(({ path, error }) => {