build: update to marked v16 and resolve breakages

Updates the repo to the latest version of `marked` and resolves some breaking changes that required us to switch some files to ESM. As a result, I had to disable syntax highlighting for type aliases and constants in the API docs, because it would've required us to rework the dgeni setup as well.

Author: crisbeto

package.json

@@ -122,7 +122,7 @@
     "karma-parallel": "^0.3.1",
     "karma-sourcemap-loader": "^0.4.0",
     "magic-string": "0.30.17",
-    "marked": "^15.0.12",
+    "marked": "^16.0.0",
     "minimatch": "^10.0.3",
     "parse5": "^7.1.2",
     "postcss": "^8.4.17",

pnpm-lock.yaml

@@ -2,7 +2,6 @@ import {Package} from 'dgeni';
 import {ReadTypeScriptModules} from 'dgeni-packages/typescript/processors/readTypeScriptModules';
 import {Host} from 'dgeni-packages/typescript/services/ts-host/host';
 import {TypeFormatFlags} from 'typescript';
-import {HighlightNunjucksExtension} from './nunjucks-tags/highlight';
 import {patchLogService} from './patch-log-service';
 import {AsyncFunctionsProcessor} from './processors/async-functions';
 import {categorizer} from './processors/categorizer';
@@ -165,6 +164,4 @@ apiDocsPackage.config(function (templateFinder: any, templateEngine: any) {
     variableStart: '{$',
     variableEnd: '$}',
   };
-
-  templateEngine.tags.push(new HighlightNunjucksExtension());
 });

tools/dgeni/docs-package.ts

@@ -17,10 +17,6 @@ <h4 id="{$ constant.name $}" class="docs-header-link docs-api-h4 docs-api-consta
 
 <div class="docs-markdown">
 <pre class="docs-markdown-pre">
-<code class="docs-markdown-code">
-{%- highlight "typescript" -%}
-  const {$ constant.name | safe $}: {$ constant.type | safe $};
-{%- end_highlight -%}
-</code>
+<code class="docs-markdown-code">const {$ constant.name | safe $}: {$ constant.type | safe $};</code>
 </pre>
 </div>

tools/dgeni/nunjucks-tags/highlight.ts

@@ -17,10 +17,6 @@ <h4 id="{$ alias.name $}" class="docs-header-link docs-api-h4 docs-api-type-alia
 
 <div class="docs-markdown">
 <pre class="docs-markdown-pre">
-<code class="docs-markdown-code">
-{%- highlight "typescript" -%}
-  type {$ alias.name | safe $} = {$ alias.typeDefinition | safe $};
-{%- end_highlight -%}
-</code>
+<code class="docs-markdown-code">type {$ alias.name | safe $} = {$ alias.typeDefinition | safe $};</code>
 </pre>
 </div>

tools/dgeni/templates/constant.template.html

@@ -5,7 +5,7 @@ package(default_visibility = ["//visibility:public"])
 
 ts_project(
     name = "extract_tokens_lib",
-    srcs = glob(["**/*.ts"]),
+    srcs = glob(["**/*.mts"]),
     tsconfig = "//tools:tsconfig",
     deps = [
         "//:node_modules/sass",
@@ -18,5 +18,5 @@ js_binary(
     data = [
         ":extract_tokens_lib",
     ],
-    entry_point = ":extract-tokens.js",
+    entry_point = ":extract-tokens.mjs",
 )

tools/dgeni/templates/type-alias.template.html

@@ -2,7 +2,7 @@ import {readFileSync, writeFileSync} from 'fs';
 import {pathToFileURL} from 'url';
 import {relative, join, dirname} from 'path';
 import {compileString} from 'sass';
-import {highlightCodeBlock} from '../highlight-files/highlight-code-block';
+import {highlightCodeBlock} from '../highlight-files/highlight-code-block.mjs';
 
 /** Information extracted for a single token from the theme. */
 interface ExtractedToken {
@@ -43,40 +43,39 @@ interface ThemeData {
 }
 
 // Script that extracts the tokens from a specific Bazel target.
-if (require.main === module) {
-  const [packagePath, outputPath, ...inputFiles] = process.argv.slice(2);
-  const themeFiles = inputFiles
-    // Filter out only the files within the package
-    // since the path also includes dependencies.
-    .filter(file => file.startsWith(packagePath))
-    .map(file => {
-      // Assumption: all theme files start with an underscore since they're
-      // partials and they end with `-theme`.
-      // Assumption: the name under which the theme mixin will be available is the
-      // same as the file name without the underscore and `-theme.scss`.
-      const match = file.match(/_(.*)-theme\.scss$/);
-      return match ? {mixinPrefix: match[1], filePath: file} : null;
-    })
-    .filter(file => !!file);
-
-  if (themeFiles.length === 0) {
-    throw new Error(`Could not find theme files in ${packagePath}`);
-  }
 
-  const themes: ThemeData[] = [];
+const [packagePath, outputPath, ...inputFiles] = process.argv.slice(2);
+const themeFiles = inputFiles
+  // Filter out only the files within the package
+  // since the path also includes dependencies.
+  .filter(file => file.startsWith(packagePath))
+  .map(file => {
+    // Assumption: all theme files start with an underscore since they're
+    // partials and they end with `-theme`.
+    // Assumption: the name under which the theme mixin will be available is the
+    // same as the file name without the underscore and `-theme.scss`.
+    const match = file.match(/_(.*)-theme\.scss$/);
+    return match ? {mixinPrefix: match[1], filePath: file} : null;
+  })
+  .filter(file => !!file);
+
+if (themeFiles.length === 0) {
+  throw new Error(`Could not find theme files in ${packagePath}`);
+}
+
+const themes: ThemeData[] = [];
 
-  themeFiles.forEach(theme => {
-    themes.push({
-      name: theme.mixinPrefix,
-      // This can be derived from the `name` already, but we want the source
-      // of truth to be in this repo, instead of whatever page consumes the data.
-      overridesMixin: `${theme.mixinPrefix}-overrides`,
-      tokens: extractTokens(theme.filePath),
-    });
+themeFiles.forEach(theme => {
+  themes.push({
+    name: theme.mixinPrefix,
+    // This can be derived from the `name` already, but we want the source
+    // of truth to be in this repo, instead of whatever page consumes the data.
+    overridesMixin: `${theme.mixinPrefix}-overrides`,
+    tokens: extractTokens(theme.filePath),
   });
+});
 
-  writeFileSync(outputPath, JSON.stringify({example: getUsageExample(themes), themes}));
-}
+writeFileSync(outputPath, JSON.stringify({example: getUsageExample(themes), themes}));
 
 /**
  * Extracts the tokens from a theme file.

tools/extract-tokens/BUILD.bazel

@@ -5,7 +5,7 @@ package(default_visibility = ["//visibility:public"])
 
 ts_project(
     name = "sources",
-    srcs = glob(["**/*.ts"]),
+    srcs = glob(["**/*.mts"]),
     tsconfig = "//tools:tsconfig",
     deps = [
         "//:node_modules/@types/fs-extra",
@@ -20,5 +20,5 @@ js_binary(
     data = [
         ":sources",
     ],
-    entry_point = ":highlight-files.js",
+    entry_point = ":highlight-files.mjs",
 )

tools/extract-tokens/extract-tokens.ts renamed to tools/extract-tokens/extract-tokens.mts

@@ -0,0 +1,73 @@
+/**
+ * Script that will be used by the highlight_files Bazel rule in order to highlight
+ * multiple input files using highlight.js. The output will be HTML files.
+ */
+
+import fsExtra from 'fs-extra';
+import {dirname, extname, join, relative} from 'path';
+import {highlightCodeBlock} from './highlight-code-block.mjs';
+import {regionParser} from '../region-parser/region-parser.mjs';
+
+/**
+ * Determines the command line arguments for the current Bazel action. Since this action can
+ * have a large set of input files, Bazel may write the arguments into a parameter file.
+ * This function is responsible for handling normal argument passing or Bazel parameter files.
+ * Read more here: https://docs.bazel.build/versions/master/skylark/lib/Args.html#use_param_file
+ */
+function getBazelActionArguments() {
+  const args = process.argv.slice(2);
+
+  // If Bazel uses a parameter file, we've specified that it passes the file in the following
+  // format: "arg0 arg1 --param-file={path_to_param_file}"
+  if (args[0].startsWith('--param-file=')) {
+    return fsExtra.readFileSync(args[0].split('=')[1], 'utf8').trim().split('\n');
+  }
+
+  return args;
+}
+
+function detectAndHighlightRegionBlocks(
+  parsed: {contents: string; regions: {[p: string]: string}},
+  basePath: string,
+  outDir: string,
+) {
+  const fileExtension = extname(basePath).substring(1);
+  for (const [regionName, regionSnippet] of Object.entries(parsed.regions)) {
+    // Create files for each found region
+    if (!regionName) {
+      continue;
+    }
+    const highlightedRegion = highlightCodeBlock(regionSnippet, fileExtension);
+    // Convert "my-component-example.ts" into "my-component-example_region-ts.html"
+    const regionBaseOutputPath = basePath.replace(
+      `.${fileExtension}`,
+      `_${regionName}-${fileExtension}.html`,
+    );
+    const regionOutputPath = join(outDir, regionBaseOutputPath);
+    fsExtra.ensureDirSync(dirname(regionOutputPath));
+    fsExtra.writeFileSync(regionOutputPath, highlightedRegion);
+  }
+}
+
+// The script expects the output directory as first argument. Second is the name of the
+// package where this the highlight target is declared. All remaining arguments will be
+// considered as markdown input files that need to be transformed.
+const [outDir, packageName, ...inputFiles] = getBazelActionArguments();
+
+// Walk through each input file and write transformed markdown output
+// to the specified output directory.
+for (const execPath of inputFiles) {
+  // Compute a relative path from the package to the actual input file.
+  // e.g `src/components-examples/cdk/<..>/example.ts` becomes `cdk/<..>/example.ts`.
+  const basePath = relative(packageName, execPath);
+  const fileExtension = extname(basePath).substring(1);
+  const parsed = regionParser(fsExtra.readFileSync(execPath, 'utf8'), fileExtension);
+  detectAndHighlightRegionBlocks(parsed, basePath, outDir);
+  // Convert "my-component-example.ts" into "my-component-example-ts.html"
+  const baseOutputPath = basePath.replace(`.${fileExtension}`, `-${fileExtension}.html`);
+  const outputPath = join(outDir, baseOutputPath);
+  const htmlOutput = highlightCodeBlock(parsed.contents, fileExtension);
+
+  fsExtra.ensureDirSync(dirname(outputPath));
+  fsExtra.writeFileSync(outputPath, htmlOutput);
+}