Refactor build logic to remove TOML dependency

Author: filmor

rustler_mix/lib/rustler.ex

@@ -79,8 +79,6 @@ defmodule Rustler do
 
     * `:target` - Specify a compile [target] triple.
 
-    * `:target_dir` - Override the compiler output directory.
-
   Any of the above options can be passed directly into the `use` macro like so:
 
       defmodule MyNIF do

rustler_mix/lib/rustler/build_results.ex

@@ -0,0 +1,63 @@
+defmodule Rustler.BuildResults do
+  defstruct artifacts: []
+end
+
+defmodule Rustler.BuildArtifact do
+  defstruct [:kind, :name, :filename]
+
+  def from_line(line) do
+    if line["reason"] == "compiler-artifact" do
+      target = line["target"]
+
+      base = %__MODULE__{
+        name: target["name"],
+        filename: line["filenames"] |> :erlang.hd()
+      }
+
+      kind = target["kind"]
+
+      maybe_lib =
+        if kind |> Enum.member?("dylib") || kind |> Enum.member?("cdylib") do
+          [%{base | kind: :lib}]
+        end || []
+
+      maybe_exec =
+        if kind |> Enum.member?("bin") do
+          [%{base | kind: :bin, filename: line["executable"]}]
+        end || []
+
+      maybe_lib ++ maybe_exec
+    end || []
+  end
+end
+
+defimpl Collectable, for: Rustler.BuildResults do
+  def into(results) do
+    maybe_log_msg = fn elem ->
+      if elem["reason"] == "compiler-message" do
+        # TODO: Log message
+        # Mix.shell().info(elem["message"]["message"])
+      end
+    end
+
+    collector_fun = fn
+      acc, {:cont, elem} ->
+        case Jason.decode(elem) do
+          {:ok, elem} ->
+            maybe_log_msg.(elem)
+            update_in(acc.artifacts, &(&1 ++ Rustler.BuildArtifact.from_line(elem)))
+
+          {:error, _err} ->
+            acc
+        end
+
+      acc, :done ->
+        acc
+
+      _acc, :halt ->
+        :ok
+    end
+
+    {results, collector_fun}
+  end
+end

rustler_mix/lib/rustler/compiler.ex

@@ -8,46 +8,76 @@ defmodule Rustler.Compiler do
     config = Config.from(otp_app, config, opts)
 
     if !config.skip_compilation? do
-      crate_full_path = Path.expand(config.path, File.cwd!())
-
       File.mkdir_p!(priv_dir())
 
       Mix.shell().info("Compiling crate #{config.crate} in #{config.mode} mode (#{config.path})")
+      do_compile(config)
+      Mix.Project.build_structure()
+
+      # See #326: Ensure that the libraries are copied into the correct subdirectory
+      # in `_build`.
+      missing = Enum.reject(expected_files(config), &File.exists?/1)
+
+      if missing != [] do
+        Mix.shell().info("Expected files do not exist: #{inspect(missing)}, rebuilding")
+
+        do_compile(config, true)
+        Mix.Project.build_structure()
+      end
+
+      missing = Enum.reject(expected_files(config), &File.exists?/1)
+
+      if missing != [] do
+        raise "Expected files do still not exist: #{inspect(missing)}\nPlease verify your configuration"
+      end
+    end
 
-      [cmd | args] =
-        make_base_command(config.cargo)
-        |> make_no_default_features_flag(config.default_features)
-        |> make_features_flag(config.features)
-        |> make_target_flag(config.target)
-        |> make_build_mode_flag(config.mode)
+    config
+  end
 
-      ensure_platform_requirements!(crate_full_path, config, :os.type())
+  defp do_compile(config, recompile \\ false) do
+    [cmd | args] =
+      rustc(config.cargo)
+      |> build_flags(config)
+      |> make_build_mode_flag(config.mode)
+
+    args =
+      if recompile do
+        # HACK: Cargo does not allow forcing a recompilation, but we need the
+        # compiler-artifact messages, so force a recompilation via a "config
+        # change"
+        args ++ ["--cfg", "build_#{System.system_time(:millisecond)}"]
+      else
+        args
+      end
 
-      compile_result =
-        System.cmd(cmd, args,
-          cd: crate_full_path,
-          stderr_to_stdout: true,
-          env: [{"CARGO_TARGET_DIR", config.target_dir} | config.env],
-          into: IO.stream(:stdio, :line)
-        )
+    compile_result =
+      System.cmd(cmd, args, env: config.env, into: %Rustler.BuildResults{}, lines: 1024)
 
+    artifacts =
       case compile_result do
-        {_, 0} -> :ok
+        {build_results, 0} -> build_results.artifacts
         {_, code} -> raise "Rust NIF compile error (rustc exit code #{code})"
       end
 
-      handle_artifacts(crate_full_path, config)
-      # See #326: Ensure that the libraries are copied into the correct subdirectory
-      # in `_build`.
-      Mix.Project.build_structure()
-    end
+    handle_artifacts(artifacts, config)
+  end
 
-    config
+  defp build_flags(args, config) do
+    args
+    |> make_package_flag(config.crate)
+    |> make_no_default_features_flag(config.default_features)
+    |> make_features_flag(config.features)
+    |> make_target_flag(config.target)
   end
 
-  defp make_base_command(:system), do: ["cargo", "rustc"]
-  defp make_base_command({:system, channel}), do: ["cargo", channel, "rustc"]
-  defp make_base_command({:bin, path}), do: [path, "rustc"]
+  defp rustc(cargo) do
+    make_base_command(cargo) ++ ["rustc", "--message-format=json-render-diagnostics", "--quiet"]
+  end
+
+  defp make_base_command(:system), do: ["cargo"]
+  defp make_base_command({:system, channel}), do: ["cargo", channel]
+  defp make_base_command({:bin, path}), do: [path]
 
   defp make_base_command({:rustup, version}) do
     if Rustup.version() == :none do
@@ -58,10 +88,10 @@ defmodule Rustler.Compiler do
       throw_error({:rust_version_not_installed, version})
     end
 
-    ["rustup", "run", version, "cargo", "rustc"]
+    ["rustup", "run", version, "cargo"]
   end
 
-  defp ensure_platform_requirements!(_, _, _), do: :ok
+  defp make_package_flag(args, crate), do: args ++ ["-p", "#{crate}"]
 
   defp make_no_default_features_flag(args, true), do: args
   defp make_no_default_features_flag(args, false), do: args ++ ["--no-default-features"]
@@ -99,71 +129,54 @@ defmodule Rustler.Compiler do
     end
   end
 
-  defp handle_artifacts(path, config) do
-    toml = toml_data(path)
+  defp handle_artifacts(artifacts, config) do
     target = config.target
-    names = get_name(toml, :lib) ++ get_name(toml, :bin)
 
-    output_dir =
-      if is_binary(target) do
-        Path.join([target, Atom.to_string(config.mode)])
-      else
-        Atom.to_string(config.mode)
-      end
+    if artifacts == [] do
+      throw_error(:no_artifacts)
+    end
 
-    Enum.each(names, fn {name, type} ->
-      {src_file, dst_file} = make_file_names(name, type, target)
-      compiled_lib = Path.join([config.target_dir, output_dir, src_file])
-      destination_lib = Path.join(priv_dir(), dst_file)
-
-      # If the file exists already, we delete it first. This is to ensure that another
-      # process, which might have the library dynamically linked in, does not generate
-      # a segfault. By deleting it first, we ensure that the copy operation below does
-      # not write into the existing file.
-      File.rm(destination_lib)
-      File.cp!(compiled_lib, destination_lib)
+    Enum.each(artifacts, fn artifact = %Rustler.BuildArtifact{} ->
+      if artifact.kind == :lib or artifact.kind == :bin do
+        dst_file = output_file(artifact.name, artifact.kind, target)
+        destination_lib = Path.join(priv_dir(), dst_file)
+
+        # If the file exists already, we delete it first. This is to ensure that another
+        # process, which might have the library dynamically linked in, does not generate
+        # a segfault. By deleting it first, we ensure that the copy operation below does
+        # not write into the existing file.
+        Mix.shell().info("Copying #{artifact.filename} to #{destination_lib}")
+        File.rm(destination_lib)
+        File.cp!(artifact.filename, destination_lib)
+      end
     end)
   end
 
-  defp get_name(toml, section) do
-    case toml[to_string(section)] do
-      nil -> []
-      values when is_map(values) -> [{values["name"], section}]
-      values when is_list(values) -> Enum.map(values, &{&1["name"], section})
-    end
-  end
+  defp expected_files(config) do
+    lib = if config.lib, do: [output_file(config.crate, :lib, config.target)], else: []
+    bins = for bin <- config.binaries, do: output_file(bin, :bin, config.target)
 
-  defp get_suffix(target, :lib) do
-    case get_target_os_type(target) do
-      {:win32, _} -> "dll"
-      {:unix, :darwin} -> "dylib"
-      {:unix, _} -> "so"
+    for filename <- lib ++ bins do
+      Path.join(priv_dir(), filename)
     end
   end
 
-  defp get_suffix(target, :bin) do
-    case get_target_os_type(target) do
-      {:win32, _} -> "exe"
-      {:unix, _} -> ""
-    end
+  defp output_file(base_name, kind, target) do
+    "#{base_name}#{suffix(kind, target)}"
   end
 
-  defp make_file_names(base_name, :lib, target) do
-    suffix = get_suffix(target, :lib)
-
+  defp suffix(:lib, target) do
     case get_target_os_type(target) do
-      {:win32, _} -> {"#{base_name}.#{suffix}", "lib#{base_name}.dll"}
-      {:unix, :darwin} -> {"lib#{base_name}.#{suffix}", "lib#{base_name}.so"}
-      {:unix, _} -> {"lib#{base_name}.#{suffix}", "lib#{base_name}.so"}
+      {:win32, _} -> ".dll"
+      {:unix, :darwin} -> ".so"
+      {:unix, _} -> ".so"
     end
   end
 
-  defp make_file_names(base_name, :bin, target) do
-    suffix = get_suffix(target, :bin)
-
+  defp suffix(:bin, target) do
     case get_target_os_type(target) do
-      {:win32, _} -> {"#{base_name}.#{suffix}", "#{base_name}.exe"}
-      {:unix, _} -> {base_name, base_name}
+      {:win32, _} -> ".exe"
+      {:unix, _} -> ""
     end
   end
 
@@ -172,19 +185,5 @@ defmodule Rustler.Compiler do
     raise "Compilation error"
   end
 
-  defp toml_data(path) do
-    if !File.dir?(path) do
-      throw_error({:nonexistent_crate_directory, path})
-    end
-
-    case File.read("#{path}/Cargo.toml") do
-      {:error, :enoent} ->
-        throw_error({:cargo_toml_not_found, path})
-
-      {:ok, text} ->
-        Toml.decode!(text)
-    end
-  end
-
   defp priv_dir, do: "priv/native"
 end

rustler_mix/lib/rustler/compiler/config.ex

@@ -18,16 +18,15 @@ defmodule Rustler.Compiler.Config do
             external_resources: [],
             features: [],
             lib: true,
+            binaries: [],
             load_data: 0,
             load_data_fun: nil,
             load_from: nil,
             mode: :release,
             otp_app: nil,
             path: "",
-            priv_dir: "",
             skip_compilation?: false,
             target: nil,
-            target_dir: "",
             metadata: nil
 
   alias Rustler.Compiler.Config
@@ -37,11 +36,10 @@ defmodule Rustler.Compiler.Config do
 
     defaults = %Config{
       crate: crate,
-      load_from: {otp_app, "priv/native/lib#{crate}"},
       mode: :release,
       otp_app: otp_app,
       path: "native/#{crate}",
-      target_dir: Application.app_dir(otp_app, "native/#{crate}")
+      load_from: {otp_app, "priv/native/#{crate}"}
     }
 
     defaults

rustler_mix/mix.exs

@@ -24,7 +24,6 @@ defmodule Rustler.Mixfile do
 
   defp deps do
     [
-      {:toml, "~> 0.7", runtime: false},
       {:ex_doc, ">= 0.0.0", only: :dev, runtime: false},
       {:jason, "~> 1.0", runtime: false}
     ]

rustler_mix/priv/templates/basic/Cargo.toml.eex

@@ -5,7 +5,6 @@ authors = []
 edition = "2021"
 
 [lib]
-name = "<%= library_name %>"
 crate-type = ["cdylib"]
 
 [dependencies]