feat: move content hashing to a child process

I noticed in profiles that this was actually a bottleneck for Jest tests
(in #214) when running as a cluster. I wanted to use a worker thread
for it, but it looks like there's an issue in vscode preventing that[1]
for the moment.

This cuts the time-to-first-breakpoint in half for the jest tests, which
is fairly nice. The child process is killed after 30 seconds of
inactivity. I may do an algorithmic optimization pass on the hash in
the future. In particular, Node/V8 now has native bigint support, which
is almost certainly faster than the `long` library.

1. microsoft/vscode#88386

Author: connor4312

gulpfile.js

@@ -189,6 +189,7 @@ async function runWebpack(packages) {
 gulp.task('package:webpack-bundle', async () => {
   const packages = [
     { entry: `${buildSrcDir}/extension.js`, library: true },
+    { entry: `${buildSrcDir}/common/hash/hash.js`, library: false },
     { entry: `${buildSrcDir}/${nodeTargetsDir}/bootloader.js`, library: false },
     { entry: `${buildSrcDir}/${nodeTargetsDir}/watchdog.js`, library: false },
   ];

src/common/hash.ts renamed to src/common/hash/hash.ts

@@ -2,8 +2,13 @@
  * Copyright (C) Microsoft Corporation. All rights reserved.
  *--------------------------------------------------------*/
 import Long from 'long';
+import { readFileRaw } from '../fsUtils';
 
-export function calculateHash(input: Buffer): string {
+/**
+ * An implementation of the Chrome content hashing algorithm used to verify
+ * whether files on disk are the same as those in the debug session.
+ */
+function calculateHash(input: Buffer): string {
   const prime = [
     new Long(0x3fb75161, 0, true),
     new Long(0xab1f4e4f, 0, true),
@@ -99,3 +104,33 @@ function normalize(buffer: Buffer): Buffer {
 function utf8ToUtf16(buffer: Buffer) {
   return Buffer.from(buffer.toString('utf8'), 'utf16le');
 }
+
+/**
+ * Message sent to the hash worker.
+ */
+export type HashRequest = { id: number; file: string } | { id: number; data: string | Buffer };
+
+/**
+ * Message received in the hash response.
+ */
+export type HashResponse = { id: number; hash?: string };
+
+function startWorker(send: (message: HashResponse) => void) {
+  process.on('message', (msg: HashRequest) => {
+    if ('file' in msg) {
+      const file = msg.file;
+      readFileRaw(file)
+        .then(data => send({ id: msg.id, hash: calculateHash(data) }))
+        .catch(() => send({ id: msg.id }));
+    } else if ('data' in msg) {
+      send({
+        id: msg.id,
+        hash: calculateHash(msg.data instanceof Buffer ? msg.data : Buffer.from(msg.data, 'utf-8')),
+      });
+    }
+  });
+}
+
+if (process.send) {
+  startWorker(process.send.bind(process));
+}

src/common/hash/index.ts

@@ -0,0 +1,53 @@
+/*---------------------------------------------------------
+ * Copyright (C) Microsoft Corporation. All rights reserved.
+ *--------------------------------------------------------*/
+
+import { ChildProcess, fork } from 'child_process';
+import { join } from 'path';
+import { HashRequest, HashResponse } from './hash';
+import { debounce } from '../objUtils';
+
+let instance: ChildProcess | undefined;
+let messageId = 0;
+
+const cleanup = debounce(30 * 1000, () => {
+  instance?.kill();
+  instance = undefined;
+});
+
+const create = () => {
+  if (instance) {
+    return instance;
+  }
+
+  instance = fork(join(__dirname, 'hash.js'), [], { env: {}, silent: true });
+  instance.setMaxListeners(Infinity);
+  return instance;
+};
+
+const send = (req: HashRequest): Promise<string | undefined> => {
+  const cp = create();
+  cleanup();
+
+  return new Promise(resolve => {
+    const listener = (res: HashResponse) => {
+      if (res.id === req.id) {
+        resolve(res.hash);
+        cp.removeListener('message', listener);
+      }
+    };
+
+    cp.addListener('message', listener);
+    cp.send(req);
+  });
+};
+
+/**
+ * Gets the Chrome content hash of script contents.
+ */
+export const hashBytes = (data: string | Buffer) => send({ data, id: messageId++ });
+
+/**
+ * Gets the Chrome content hash of a file.
+ */
+export const hashFile = (file: string) => send({ file, id: messageId++ });

src/common/objUtils.ts

@@ -172,7 +172,7 @@ export function debounce(duration: number, fn: () => void): (() => void) & { cle
   let timeout: NodeJS.Timer | void;
   const debounced = () => {
     if (timeout !== undefined) {
-      return;
+      clearTimeout(timeout);
     }
 
     timeout = setTimeout(() => {

src/common/sourceUtils.ts

@@ -7,10 +7,10 @@ import * as sourceMap from 'source-map';
 import * as ts from 'typescript';
 import * as urlUtils from './urlUtils';
 import * as fsUtils from './fsUtils';
-import { calculateHash } from './hash';
 import { SourceMap, ISourceMapMetadata } from './sourceMaps/sourceMap';
 import { logger } from './logging/logger';
 import { LogTag } from './logging';
+import { hashBytes, hashFile } from './hash';
 
 export async function prettyPrintAsSourceMap(
   fileName: string,
@@ -310,12 +310,11 @@ export async function checkContentHash(
     const exists = await fsUtils.exists(absolutePath);
     return exists ? absolutePath : undefined;
   }
-  const content =
+  const hash =
     typeof contentOverride === 'string'
-      ? Buffer.from(contentOverride, 'utf8')
-      : await fsUtils.readFileRaw(absolutePath);
+      ? await hashBytes(contentOverride)
+      : await hashFile(absolutePath);
 
-  const hash = calculateHash(content);
   return hash === contentHash ? absolutePath : undefined;
 }
 

src/test/sources/sources-hash-from-file.txt

@@ -0,0 +1,4 @@
+1d9f277f134f31935a286ff810acdf571af3498e
+1d9f277f134f31935a286ff810acdf571af3498e
+1d9f277f134f31935a286ff810acdf571af3498e
+1d9f277f134f31935a286ff810acdf571af3498e

src/test/sources/sourcesTest.ts

@@ -2,10 +2,11 @@
  * Copyright (C) Microsoft Corporation. All rights reserved.
  *--------------------------------------------------------*/
 
-import { TestP } from '../test';
+import { TestP, createFileTree, testFixturesDir } from '../test';
 import Dap from '../../dap/api';
-import { calculateHash } from '../../common/hash';
+import { hashBytes, hashFile } from '../../common/hash';
 import { itIntegrates } from '../testIntegrationUtils';
+import { join } from 'path';
 
 describe('sources', () => {
   async function dumpSource(p: TestP, event: Dap.LoadedSourceEventParams, name: string) {
@@ -149,10 +150,25 @@ describe('sources', () => {
     0x31, 0x00, 0x31, 0x00, 0x31, 0x00, 0x22, 0x00]);
 
   itIntegrates('hash bom', async ({ r }) => {
-    r.log(calculateHash(utf8NoBOM));
-    r.log(calculateHash(utf8BOM));
-    r.log(calculateHash(utf16BigEndianBOM));
-    r.log(calculateHash(utf16LittleEndianBOM));
+    r.log(await hashBytes(utf8NoBOM));
+    r.log(await hashBytes(utf8BOM));
+    r.log(await hashBytes(utf16BigEndianBOM));
+    r.log(await hashBytes(utf16LittleEndianBOM));
+    r.assertLog();
+  });
+
+  itIntegrates('hash from file', async ({ r }) => {
+    createFileTree(testFixturesDir, {
+      utf8NoBOM,
+      utf8BOM,
+      utf16BigEndianBOM,
+      utf16LittleEndianBOM,
+    });
+
+    r.log(await hashFile(join(testFixturesDir, 'utf8NoBOM')));
+    r.log(await hashFile(join(testFixturesDir, 'utf8BOM')));
+    r.log(await hashFile(join(testFixturesDir, 'utf16BigEndianBOM')));
+    r.log(await hashFile(join(testFixturesDir, 'utf16LittleEndianBOM')));
     r.assertLog();
   });
 
@@ -173,7 +189,7 @@ describe('sources', () => {
   0x75, 0x72, 0x6E, 0x20, 0x32, 0x35, 0x3B, 0x0D, 0x0A, 0x7D]);
 
   itIntegrates('hash code points', async ({ r }) => {
-    r.log(calculateHash(multiByteCodePoints));
+    r.log(await hashBytes(multiByteCodePoints.toString('utf-8')));
     r.assertLog();
   });
 });