[perf] Install all packages to profile in a single nix command (#2130)

## Summary

This change installs all packages into profile with same command. If we
fail due to priority conflict, we fallback to the previous one by one
installation.

This significantly improves performance on projects with many packages
when all packages are already in store. This aims to solve a common CICD
case where nothing has changed, all packages are in store, but we need
to create the .devbox directory and nix profile from scratch.

cc: @Lagoja this helps address some reported issues.

## How was it tested?

Happy path:
On project with 30+ packages, delete `.devbox` directory and ran `devbox
install`. This change reduced total time from 10s to 2s.

Sad path:
On new project added `curl`, `ruby`, `bundler`, `go`. `ruby` and
`bundler` conflict. I ran `devbox install` and it correctly fell back to
one by one. I inspected the profile to ensure nix did not install
anything twice.

Author: mikeland73

internal/devbox/nixprofile.go

@@ -2,6 +2,7 @@ package devbox
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"strings"
@@ -62,22 +63,24 @@ func (d *Devbox) syncNixProfileFromFlake(ctx context.Context) error {
 		}
 	}
 	if len(add) > 0 {
-		// We need to install the packages in the nix profile one-by-one because
-		// we do checks for insecure packages.
-		// TODO: move the insecure package check here, and do `nix profile install installables...`
-		// in one command for speed.
-		for _, addPath := range add {
-			if err = nix.ProfileInstall(ctx, &nix.ProfileInstallArgs{
-				Installable: addPath,
-				// Install in offline mode for speed. We know we should have all the files
-				// locally in /nix/store since we have run `nix print-dev-env` prior to this.
-				// Also avoids some "substituter not found for store-path" errors.
-				Offline:     true,
-				ProfilePath: profilePath,
-				Writer:      d.stderr,
-			}); err != nil {
-				return fmt.Errorf("error installing package in nix profile %s: %w", addPath, err)
+		if err = nix.ProfileInstall(ctx, &nix.ProfileInstallArgs{
+			Installables: add,
+			ProfilePath:  profilePath,
+			Writer:       d.stderr,
+		}); errors.Is(err, nix.ErrPriorityConflict) {
+			// We need to install the packages one by one because there was possibly a priority conflict
+			// This is slower, but uncommon.
+			for _, addPath := range add {
+				if err = nix.ProfileInstall(ctx, &nix.ProfileInstallArgs{
+					Installables: []string{addPath},
+					ProfilePath:  profilePath,
+					Writer:       d.stderr,
+				}); err != nil {
+					return fmt.Errorf("error installing package in nix profile %s: %w", addPath, err)
+				}
 			}
+		} else if err != nil {
+			return fmt.Errorf("error installing packages in nix profile %s: %w", add, err)
 		}
 	}
 	return nil

internal/nix/eval.go

@@ -55,10 +55,6 @@ func Eval(path string) ([]byte, error) {
 	return cmd.CombinedOutput(context.TODO())
 }
 
-func AllowInsecurePackages() {
-	os.Setenv("NIXPKGS_ALLOW_INSECURE", "1")
-}
-
 func IsInsecureAllowed() bool {
 	allowed, _ := strconv.ParseBool(os.Getenv("NIXPKGS_ALLOW_INSECURE"))
 	return allowed

internal/nix/profiles.go

@@ -4,6 +4,7 @@
 package nix
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -14,7 +15,6 @@ import (
 	"path/filepath"
 
 	"github.com/pkg/errors"
-	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/debug"
 	"go.jetpack.io/devbox/internal/redact"
 )
@@ -32,48 +32,43 @@ func ProfileList(writer io.Writer, profilePath string, useJSON bool) (string, er
 }
 
 type ProfileInstallArgs struct {
-	Installable string
-	Offline     bool
-	ProfilePath string
-	Writer      io.Writer
+	Installables []string
+	ProfilePath  string
+	Writer       io.Writer
 }
 
+var ErrPriorityConflict = errors.New("priority conflict")
+
 func ProfileInstall(ctx context.Context, args *ProfileInstallArgs) error {
 	defer debug.FunctionTimer().End()
-	if !IsInsecureAllowed() && PackageIsInsecure(args.Installable) {
-		knownVulnerabilities := PackageKnownVulnerabilities(args.Installable)
-		errString := fmt.Sprintf("Package %s is insecure. \n\n", args.Installable)
-		if len(knownVulnerabilities) > 0 {
-			errString += fmt.Sprintf("Known vulnerabilities: %s \n\n", knownVulnerabilities)
-		}
-		errString += "To override use `devbox add <pkg> --allow-insecure`"
-		return usererr.New(errString)
-	}
 
 	cmd := command(
 		"profile", "install",
 		"--profile", args.ProfilePath,
-		"--impure", // for NIXPKGS_ALLOW_UNFREE
+		"--offline", // makes it faster. Package is already in store
+		"--impure",  // for NIXPKGS_ALLOW_UNFREE
 		// Using an arbitrary priority to avoid conflicts with other packages.
 		// Note that this is not really the priority we care about, since we
 		// use the flake.nix to specify the priority.
 		"--priority", nextPriority(args.ProfilePath),
 	)
-	if args.Offline {
-		cmd.Args = append(cmd.Args, "--offline")
-	}
-	cmd.Args = append(cmd.Args, args.Installable)
+
+	cmd.Args = appendArgs(cmd.Args, args.Installables)
 	cmd.Env = allowUnfreeEnv(os.Environ())
 
-	// If nix profile install runs as tty, the output is much nicer. If we ever
-	// need to change this to our own writers, consider that you may need
-	// to implement your own nicer output. --print-build-logs flag may be useful.
-	cmd.Stdin = os.Stdin
-	cmd.Stdout = args.Writer
-	cmd.Stderr = args.Writer
+	// We used to attach this function to stdout and in in order to get the more interactive output.
+	// However, now we do the building in nix.Build, by the time we install in profile everything
+	// should already be in the store. We need to capture the output so we can decide if a conflict
+	// happened.
 
 	slog.Debug("running command", "cmd", cmd)
-	return cmd.Run(ctx)
+	out, err := cmd.CombinedOutput(ctx)
+
+	if bytes.Contains(out, []byte("error: An existing package already provides the following file")) {
+		return ErrPriorityConflict
+	}
+
+	return err
 }
 
 // ProfileRemove removes packages from a profile.

testscripts/add/add_insecure.tst.txt

@@ -0,0 +1,9 @@
+# Tests installing an insecure package.
+# This test is pretty slow, maybe there's a different package we can
+# use for testing.
+
+# we could also isolate this test and run on its own.
+
+exec devbox init
+exec devbox add [email protected] --allow-insecure python-2.7.18.6
+exec devbox install