[CAS] Allow uncached job from CAS based dependency scanning

Create a path that swift-frontend can execute an uncached job from
modules built with CAS based explicit module build. The new flag
-import-module-from-cas will allow an uncached build to load module
from CAS, and combined with source file from real file system to build
the current module. This allows quick iterations that bypasses CAS,
without full dependency scanning every time in between.

rdar://152441866

Author: cachemeifyoucan

include/swift/Basic/CASOptions.h

@@ -34,6 +34,9 @@ class CASOptions final {
   /// Skip replaying outputs from cache.
   bool CacheSkipReplay = false;
 
+  /// Import modules from CAS.
+  bool ImportModuleFromCAS = false;
+
   /// CASOptions
   clang::CASOptions CASOpts;
 

include/swift/Frontend/Frontend.h

@@ -176,7 +176,8 @@ class CompilerInvocation {
   }
 
   bool requiresCAS() const {
-    return CASOpts.EnableCaching || IRGenOpts.UseCASBackend;
+    return CASOpts.EnableCaching || IRGenOpts.UseCASBackend ||
+           CASOpts.ImportModuleFromCAS;
   }
 
   void setClangModuleCachePath(StringRef Path) {

include/swift/Option/FrontendOptions.td

@@ -244,6 +244,9 @@ def module_can_import_version: MultiArg<["-"], "module-can-import-version", 3>,
   MetaVarName<"<moduleName> <version> <underlyingVersion>">,
   HelpText<"Specify canImport module and versions">;
 
+def module_import_from_cas: Flag<["-"], "module-import-from-cas">,
+  HelpText<"Import modules from CAS instead of file system">;
+
 def disable_cross_import_overlay_search: Flag<["-"], "disable-cross-import-overlay-search">,
   HelpText<"Disable searching for cross import overlay file">;
 

lib/ClangImporter/ClangImporter.cpp

@@ -1161,7 +1161,7 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
     // to reduce the number of argument passing on the command-line and swift
     // compiler can be more efficient to compute swift cache key without having
     // the knowledge about clang command-line options.
-    if (ctx.CASOpts.EnableCaching) {
+    if (ctx.CASOpts.EnableCaching || ctx.CASOpts.ImportModuleFromCAS) {
       CI->getCASOpts() = ctx.CASOpts.CASOpts;
       // When clangImporter is used to compile (generate .pcm or .pch), need to
       // inherit the include tree from swift args (last one wins) and clear the

lib/Frontend/CompilerInvocation.cpp

@@ -800,6 +800,8 @@ static bool ParseCASArgs(CASOptions &Opts, ArgList &Args,
                                             std::string(Value));
   }
 
+  Opts.ImportModuleFromCAS |= Args.hasArg(OPT_module_import_from_cas);
+
   if (auto *A = Args.getLastArg(OPT_clang_include_tree_root))
     Opts.ClangIncludeTree = A->getValue();
   if (auto *A = Args.getLastArg(OPT_clang_include_tree_filelist))

lib/Frontend/Frontend.cpp

@@ -821,7 +821,8 @@ bool CompilerInstance::setUpModuleLoaders() {
   if (ExplicitModuleBuild ||
       !Invocation.getSearchPathOptions().ExplicitSwiftModuleMapPath.empty() ||
       !Invocation.getSearchPathOptions().ExplicitSwiftModuleInputs.empty()) {
-    if (Invocation.getCASOptions().EnableCaching)
+    if (Invocation.getCASOptions().EnableCaching ||
+        Invocation.getCASOptions().ImportModuleFromCAS)
       ESML = ExplicitCASModuleLoader::create(
           *Context, getObjectStore(), getActionCache(), getDependencyTracker(),
           MLM, Invocation.getSearchPathOptions().ExplicitSwiftModuleMapPath,

test/CAS/uncached-casfs.swift

@@ -0,0 +1,79 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: %target-swift-frontend -scan-dependencies -module-name Test -O -module-cache-path %t/clang-module-cache \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   -import-objc-header %t/base/Bridging.h -scanner-output-dir %t -auto-bridging-header-chaining -scanner-debug-write-output \
+// RUN:   %t/base/test.swift %t/base/foo.swift -I %t/include -o %t/deps.json -cache-compile-job -cas-path %t/cas
+
+// RUN: %{python} %S/../CAS/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/shim.cmd
+// RUN: %swift_frontend_plain @%t/shim.cmd
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:Dummy > %t/dummy.cmd
+// RUN: %swift_frontend_plain @%t/dummy.cmd
+
+// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json
+// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid
+
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json bridgingHeader > %t/header.cmd
+// RUN: %target-swift-frontend @%t/header.cmd -disable-implicit-swift-modules -O -o %t/bridging.pch
+// RUN: %cache-tool -cas-path %t/cas -cache-tool-action print-output-keys -- \
+// RUN:   %target-swift-frontend @%t/header.cmd -disable-implicit-swift-modules -O -o %t/bridging.pch > %t/keys.json
+// RUN: %{python} %S/Inputs/ExtractOutputKey.py %t/keys.json > %t/key
+
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd
+// RUN: echo "\"-disable-implicit-string-processing-module-import\"" >> %t/MyApp.cmd
+// RUN: echo "\"-disable-implicit-concurrency-module-import\"" >> %t/MyApp.cmd
+// RUN: echo "\"-disable-implicit-swift-modules\"" >> %t/MyApp.cmd
+// RUN: echo "\"-import-objc-header\"" >> %t/MyApp.cmd
+// RUN: echo "\"%t/base/Bridging.h\"" >> %t/MyApp.cmd
+// RUN: echo "\"-import-pch\"" >> %t/MyApp.cmd
+// RUN: echo "\"%t/bridging.pch\"" >> %t/MyApp.cmd
+// RUN: echo "\"-bridging-header-pch-key\"" >> %t/MyApp.cmd
+// RUN: echo "\"@%t/key\"" >> %t/MyApp.cmd
+// RUN: echo "\"-explicit-swift-module-map-file\"" >> %t/MyApp.cmd
+// RUN: echo "\"@%t/map.casid\"" >> %t/MyApp.cmd
+
+// RUN: sed -e "s@VFS_DIR@%{/t:regex_replacement}/base@g" -e "s@EXTERNAL_DIR@%{/t:regex_replacement}/modified@g" %t/base.yaml > %t/overlay.yaml
+
+// RUN: %target-swift-frontend %t/base/test.swift %t/base/foo.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c \
+// RUN:  -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd -vfsoverlay %t/overlay.yaml -module-import-from-cas
+
+
+//--- base/test.swift
+import Dummy
+public func testFunc() {
+  non_existing_func()
+}
+
+//--- base/foo.swift
+public func foo() {}
+
+//--- modified/test.swift
+import Dummy
+public func testFunc() {
+  dummy()
+  bridge()
+}
+
+//--- base/Bridging.h
+void bridge(void);
+
+//--- include/module.modulemap
+module Dummy {
+ umbrella header "Dummy.h"
+}
+
+//--- include/Dummy.h
+void dummy(void);
+
+//--- base.yaml
+{
+  version: 0,
+  roots: [
+    {
+      type: "directory-remap",
+      name: "VFS_DIR",
+      external-contents: "EXTERNAL_DIR"
+    }
+  ]
+}