Add support for variants targets (#61)

* Make build of metadata save the variants

This is needed in order to load the correct variants and building the
list of artifacts for download.

* Validates variants in the config

* Add variants to the "available_nif_urls" function

This makes the mix task know that needs to download the variants as
well.

* Testing for available NIF urls

* Fix mix task that downloads artifacts for current target

* Fix with suggestions from PR

Author: philss

lib/mix/tasks/rustler_precompiled.download.ex

@@ -39,7 +39,7 @@ defmodule Mix.Tasks.RustlerPrecompiled.Download do
           RustlerPrecompiled.available_nif_urls(module)
 
         Keyword.get(options, :only_local) ->
-          [RustlerPrecompiled.current_target_nif_url(module)]
+          RustlerPrecompiled.current_target_nif_urls(module)
 
         true ->
           raise "you need to specify either \"--all\" or \"--only-local\" flags"

lib/rustler_precompiled.ex

@@ -66,6 +66,15 @@ defmodule RustlerPrecompiled do
     * `:max_retries` - The maximum of retries before giving up. Defaults to `3`.
       Retries can be disabled with `0`.
 
+    * `:variants` - A map with alternative versions of a given target. This is useful to
+      support specific versions of dependencies, such as an old glibc version, or to support
+      restrict CPU features, like AVX on x86_64.
+
+      The order of variants matters, because the first one that returns `true` is going to be
+      selected. Example: 
+
+          %{"x86_64-unknown-linux-gnu" => [old_glibc: fn _config -> has_old_glibc?() end]}
+
   In case "force build" is used, all options except `:base_url`, `:version`,
   `:force_build`, `:nif_versions`, and `:targets` are going to be passed down to `Rustler`.
   So if you need to configure the build, check the `Rustler` options.
@@ -180,7 +189,8 @@ defmodule RustlerPrecompiled do
               :force_build,
               :targets,
               :nif_versions,
-              :max_retries
+              :max_retries,
+              :variants
             ])
 
           {:force_build, rustler_opts}
@@ -225,11 +235,23 @@ defmodule RustlerPrecompiled do
   is stored in a metadata file.
   """
   def available_nif_urls(nif_module) when is_atom(nif_module) do
-    metadata =
-      nif_module
-      |> metadata_file()
-      |> read_map_from_file()
+    nif_module
+    |> metadata_file()
+    |> read_map_from_file()
+    |> nif_urls_from_metadata()
+    |> case do
+      {:ok, urls} ->
+        urls
+
+      {:error, wrong_meta} ->
+        raise "metadata about current target for the module #{inspect(nif_module)} is not available. " <>
+                "Please compile the project again with: `mix compile --force` " <>
+                "Metadata found: #{inspect(wrong_meta, limit: :infinity, pretty: true)}"
+    end
+  end
 
+  @doc false
+  def nif_urls_from_metadata(metadata) when is_map(metadata) do
     case metadata do
       %{
         targets: targets,
@@ -238,42 +260,73 @@ defmodule RustlerPrecompiled do
         nif_versions: nif_versions,
         version: version
       } ->
-        for target_triple <- targets, nif_version <- nif_versions do
-          target = "nif-#{nif_version}-#{target_triple}"
+        all_tar_gzs =
+          for target_triple <- targets, nif_version <- nif_versions do
+            target = "nif-#{nif_version}-#{target_triple}"
 
-          # We need to build again the name because each arch is different.
-          lib_name = "#{lib_prefix(target)}#{basename}-v#{version}-#{target}"
+            # We need to build again the name because each arch is different.
+            lib_name = "#{lib_prefix(target)}#{basename}-v#{version}-#{target}"
+            file_name = lib_name_with_ext(target_triple, lib_name)
 
-          tar_gz_file_url(base_url, lib_name_with_ext(target, lib_name))
-        end
+            tar_gz_urls(base_url, file_name, target_triple, metadata[:variants])
+          end
 
-      _ ->
-        raise "metadata about current target for the module #{inspect(nif_module)} is not available. " <>
-                "Please compile the project again with: `mix compile --force`"
+        {:ok, List.flatten(all_tar_gzs)}
+
+      wrong_meta ->
+        {:error, wrong_meta}
     end
   end
 
+  defp maybe_variants_tar_gz_urls(nil, _, _, _), do: []
+
+  defp maybe_variants_tar_gz_urls(variants, base_url, target_triple, lib_name)
+       when is_map_key(variants, target_triple) do
+    variants = Map.fetch!(variants, target_triple)
+
+    for variant <- variants do
+      tar_gz_file_url(
+        base_url,
+        lib_name_with_ext(target_triple, lib_name <> "--" <> Atom.to_string(variant))
+      )
+    end
+  end
+
+  defp maybe_variants_tar_gz_urls(_, _, _, _), do: []
+
   @doc """
-  Returns the file URL to be downloaded for current target.
+  Returns the file URLs to be downloaded for current target.
 
+  It is in the plural because a target may have some variants for it.
   It receives the NIF module.
   """
-  def current_target_nif_url(nif_module) when is_atom(nif_module) do
+  def current_target_nif_urls(nif_module) when is_atom(nif_module) do
     metadata =
       nif_module
       |> metadata_file()
       |> read_map_from_file()
 
     case metadata do
       %{base_url: base_url, file_name: file_name} ->
-        tar_gz_file_url(base_url, file_name)
+        target_triple = target_triple_from_nif_target(metadata[:target])
+
+        tar_gz_urls(base_url, file_name, target_triple, metadata[:variants])
 
       _ ->
         raise "metadata about current target for the module #{inspect(nif_module)} is not available. " <>
                 "Please compile the project again with: `mix compile --force`"
     end
   end
 
+  defp tar_gz_urls(base_url, file_name, target_triple, variants) do
+    [lib_name, _] = String.split(file_name, ".", parts: 2)
+
+    [
+      tar_gz_file_url(base_url, file_name)
+      | maybe_variants_tar_gz_urls(variants, base_url, target_triple, lib_name)
+    ]
+  end
+
   @doc """
   Returns the target triple for download or compile and load.
 
@@ -501,14 +554,19 @@ defmodule RustlerPrecompiled do
       crate: config.crate,
       otp_app: config.otp_app,
       targets: config.targets,
+      variants: variants_for_metadata(config.variants),
       nif_versions: config.nif_versions,
       version: config.version
     }
 
     case target(target_config(config.nif_versions), config.targets, config.nif_versions) do
       {:ok, target} ->
         basename = config.crate || config.otp_app
-        lib_name = "#{lib_prefix(target)}#{basename}-v#{config.version}-#{target}"
+
+        target_triple = target_triple_from_nif_target(target)
+
+        variant = variant_suffix(target_triple, config)
+        lib_name = "#{lib_prefix(target)}#{basename}-v#{config.version}-#{target}#{variant}"
 
         file_name = lib_name_with_ext(target, lib_name)
 
@@ -534,6 +592,38 @@ defmodule RustlerPrecompiled do
     end
   end
 
+  defp variants_for_metadata(variants) do
+    Map.new(variants, fn {target, values} -> {target, Keyword.keys(values)} end)
+  end
+
+  # Extract the target without the nif-NIF-VERSION part
+  defp target_triple_from_nif_target(nif_target) do
+    ["nif", _version, triple] = String.split(nif_target, "-", parts: 3)
+    triple
+  end
+
+  defp variant_suffix(target, %{variants: variants} = config) when is_map_key(variants, target) do
+    variants = Map.fetch!(variants, target)
+
+    callback = fn {_name, func} ->
+      if is_function(func, 1) do
+        func.(config)
+      else
+        func.()
+      end
+    end
+
+    case Enum.find(variants, callback) do
+      {name, _} ->
+        "--" <> Atom.to_string(name)
+
+      nil ->
+        ""
+    end
+  end
+
+  defp variant_suffix(_, _), do: ""
+
   # Perform the download or load of the precompiled NIF
   # It will look in the "priv/native/otp_app" first, and if
   # that file doesn't exist, it will try to fetch from cache.

lib/rustler_precompiled/config.ex

@@ -13,6 +13,7 @@ defmodule RustlerPrecompiled.Config do
     :force_build?,
     :targets,
     :nif_versions,
+    variants: %{},
     max_retries: 3
   ]
 
@@ -65,6 +66,7 @@ defmodule RustlerPrecompiled.Config do
       base_cache_dir: opts[:base_cache_dir],
       targets: targets,
       nif_versions: nif_versions,
+      variants: validate_variants!(targets, Keyword.get(opts, :variants, %{})),
       max_retries: validate_max_retries!(Keyword.get(opts, :max_retries, 3))
     }
   end
@@ -125,4 +127,30 @@ defmodule RustlerPrecompiled.Config do
   end
 
   defp pre_release?(version), do: "dev" in Version.parse!(version).pre
+
+  defp validate_variants!(_, nil), do: %{}
+
+  defp validate_variants!(targets, variants) when is_map(variants) do
+    variants_targets = Map.keys(variants)
+
+    for target <- variants_targets do
+      if target not in targets do
+        raise "`:variants` contains a target that is not in the list of valid targets: #{inspect(target)}"
+      end
+
+      possibilities = Map.fetch!(variants, target)
+
+      for {name, fun} <- possibilities do
+        if not is_atom(name) do
+          raise "`:variants` expects a keyword list as values, but found a key that is not an atom: #{inspect(name)}"
+        end
+
+        if not (is_function(fun, 0) or is_function(fun, 1)) do
+          raise "`:variants` expects a keyword list as values with functions to detect if a given variant is to be activated"
+        end
+      end
+    end
+
+    variants
+  end
 end

test/rustler_precompiled/config_test.exs

@@ -208,4 +208,54 @@ defmodule RustlerPrecompiled.ConfigTest do
     opts = Keyword.update!(opts, :max_retries, fn _ -> nil end)
     assert_raise RuntimeError, fn -> Config.new(opts) end
   end
+
+  test "new/1 validates variants" do
+    variants = %{"x86_64-unknown-linux-gnu" => [old_glibc: fn _config -> true end]}
+
+    opts = [
+      otp_app: :rustler_precompiled,
+      module: RustlerPrecompilationExample.Native,
+      base_url:
+        "https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0",
+      variants: variants,
+      version: "0.2.0-dev"
+    ]
+
+    assert Config.new(opts).variants == variants
+
+    zero_arity_variants = %{"x86_64-unknown-linux-gnu" => [old_glibc: fn -> true end]}
+    opts = Keyword.update!(opts, :variants, fn _ -> zero_arity_variants end)
+
+    assert Config.new(opts).variants == zero_arity_variants
+
+    opts = Keyword.update!(opts, :variants, fn _ -> nil end)
+    assert Config.new(opts).variants == %{}
+
+    invalid_target_in_variants = %{"x86_64-unknown-lizzard-hurd" => [old_glibc: fn -> true end]}
+    opts = Keyword.update!(opts, :variants, fn _ -> invalid_target_in_variants end)
+
+    error_msg =
+      ~s|`:variants` contains a target that is not in the list of valid targets: "x86_64-unknown-lizzard-hurd"|
+
+    assert_raise RuntimeError, error_msg, fn -> Config.new(opts) end
+
+    more_than_one_arity_variants = %{
+      "x86_64-unknown-linux-gnu" => [old_glibc: fn _config, _foo -> true end]
+    }
+
+    opts = Keyword.update!(opts, :variants, fn _ -> more_than_one_arity_variants end)
+
+    error_msg =
+      "`:variants` expects a keyword list as values with functions to detect if a given variant is to be activated"
+
+    assert_raise RuntimeError, error_msg, fn -> Config.new(opts) end
+
+    variants_without_keywords = %{"x86_64-unknown-linux-gnu" => [{"old_glibc", fn -> true end}]}
+    opts = Keyword.update!(opts, :variants, fn _ -> variants_without_keywords end)
+
+    error_msg =
+      ~s|`:variants` expects a keyword list as values, but found a key that is not an atom: "old_glibc"|
+
+    assert_raise RuntimeError, error_msg, fn -> Config.new(opts) end
+  end
 end