Optimize resolver usage

Author: matteius

Pipfile

@@ -31,6 +31,7 @@ zipp = "==3.21.0"
 
 [packages]
 pytz = "*"
+myproject = {ref = "main", git = "https://\\:****@github.com/user/myproject.git"}
 
 [scripts]
 tests = "bash ./run-tests.sh"

benchmarks/benchmark.py

@@ -7,6 +7,7 @@
 import subprocess
 import sys
 import time
+import urllib.request
 from pathlib import Path
 from typing import List, Tuple
 
@@ -68,8 +69,6 @@ def run_timed_command(
     def setup_requirements(self):
         """Download and prepare requirements.txt."""
         print("Setting up requirements.txt...")
-        import urllib.request
-
         requirements_path = self.benchmark_dir / "requirements.txt"
 
         try:

pipenv/cli/options.py

@@ -3,6 +3,7 @@
 
 from pipenv.project import Project
 from pipenv.utils import console, err
+from pipenv.utils.display import format_help
 from pipenv.utils.internet import is_valid_url
 from pipenv.vendor.click import (
     BadArgumentUsage,
@@ -23,8 +24,6 @@ class PipenvGroup(DYMMixin, Group):
     """Custom Group class provides formatted main help"""
 
     def get_help_option(self, ctx):
-        from pipenv.utils.display import format_help
-
         """Override for showing formatted main help via --help and -h options"""
         help_options = self.get_help_option_names(ctx)
         if not help_options or not self.add_help_option:

pipenv/environment.py

@@ -8,6 +8,7 @@
 import sys
 import typing
 from functools import cached_property
+from itertools import chain
 from pathlib import Path
 from sysconfig import get_paths, get_python_version, get_scheme_names
 from urllib.parse import urlparse
@@ -16,8 +17,10 @@
 from pipenv.patched.pip._internal.commands.install import InstallCommand
 from pipenv.patched.pip._internal.index.package_finder import PackageFinder
 from pipenv.patched.pip._internal.req.req_install import InstallRequirement
+from pipenv.patched.pip._vendor.packaging.markers import UndefinedEnvironmentName
 from pipenv.patched.pip._vendor.packaging.specifiers import SpecifierSet
 from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name
+from pipenv.patched.pip._vendor.packaging.version import Version
 from pipenv.patched.pip._vendor.packaging.version import parse as parse_version
 from pipenv.patched.pip._vendor.typing_extensions import Iterable
 from pipenv.utils import console
@@ -28,6 +31,8 @@
 from pipenv.utils.shell import temp_environ
 from pipenv.utils.virtualenv import virtualenv_scripts_dir
 from pipenv.vendor.importlib_metadata.compat.py39 import normalized_name
+from pipenv.vendor.pipdeptree._models.dag import PackageDAG
+from pipenv.vendor.pipdeptree._models.package import InvalidRequirementError
 from pipenv.vendor.pythonfinder.utils import is_in_path
 
 if sys.version_info < (3, 10):
@@ -102,8 +107,6 @@ def safe_import(self, name: str) -> ModuleType:
     def python_version(self) -> str | None:
         with self.activated() as active:
             if active:
-                from pipenv.patched.pip._vendor.packaging.version import Version
-
                 # Extract version parts
                 version_str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
                 python_version = Version(version_str)  # Create PEP 440 compliant version
@@ -633,12 +636,6 @@ def _get_requirements_for_package(cls, node, key_tree, parent=None, chain=None):
         return d
 
     def get_package_requirements(self, pkg=None):
-        from itertools import chain
-
-        from pipenv.patched.pip._vendor.packaging.markers import UndefinedEnvironmentName
-        from pipenv.vendor.pipdeptree._models.dag import PackageDAG
-        from pipenv.vendor.pipdeptree._models.package import InvalidRequirementError
-
         flatten = chain.from_iterable
 
         packages = self.get_installed_packages()

pipenv/project.py

@@ -1273,6 +1273,100 @@ def add_pipfile_entry_to_pipfile(self, name, normalized_name, entry, category=No
         self.write_toml(parsed_pipfile)
         return newly_added, category, normalized_name
 
+    def add_packages_to_pipfile_batch(self, packages_data, dev=False, categories=None):
+        """
+        Add multiple packages to Pipfile in a single operation for better performance.
+
+        Args:
+            packages_data: List of tuples (package, pip_line) or list of dicts with package info
+            dev: Whether to add to dev-packages section
+            categories: List of categories to add packages to
+
+        Returns:
+            List of tuples (newly_added, category, normalized_name) for each package
+        """
+        if not packages_data:
+            return []
+
+        # Determine target categories
+        if categories is None or (isinstance(categories, list) and len(categories) == 0):
+            categories = ["dev-packages" if dev else "packages"]
+        elif isinstance(categories, str):
+            categories = [categories]
+
+        # Read Pipfile once
+        parsed_pipfile = self.parsed_pipfile
+        results = []
+
+        # Ensure all categories exist
+        for category in categories:
+            if category not in parsed_pipfile:
+                parsed_pipfile[category] = {}
+
+        # Process all packages
+        for package_data in packages_data:
+            if isinstance(package_data, tuple) and len(package_data) == 2:
+                package, pip_line = package_data
+
+                # Generate entry for this package
+                name, normalized_name, entry = self.generate_package_pipfile_entry(
+                    package, pip_line, category=categories[0]
+                )
+
+                # Add to each specified category
+                for category in categories:
+                    newly_added = False
+
+                    # Remove any existing entries with different casing
+                    section = parsed_pipfile.get(category, {})
+                    for entry_name in section.copy().keys():
+                        if entry_name.lower() == normalized_name.lower():
+                            del parsed_pipfile[category][entry_name]
+
+                    # Check if this is a new package
+                    if normalized_name not in parsed_pipfile[category]:
+                        newly_added = True
+
+                    # Add the package
+                    parsed_pipfile[category][normalized_name] = entry
+                    results.append((newly_added, category, normalized_name))
+
+            elif isinstance(package_data, dict):
+                # Handle pre-processed package data
+                name = package_data.get("name")
+                normalized_name = package_data.get("normalized_name")
+                entry = package_data.get("entry")
+
+                if name and normalized_name and entry:
+                    for category in categories:
+                        newly_added = False
+
+                        # Remove any existing entries with different casing
+                        section = parsed_pipfile.get(category, {})
+                        for entry_name in section.copy().keys():
+                            if entry_name.lower() == normalized_name.lower():
+                                del parsed_pipfile[category][entry_name]
+
+                        # Check if this is a new package
+                        if normalized_name not in parsed_pipfile[category]:
+                            newly_added = True
+
+                        # Add the package
+                        parsed_pipfile[category][normalized_name] = entry
+                        results.append((newly_added, category, normalized_name))
+
+        # Sort categories if requested
+        if self.settings.get("sort_pipfile"):
+            for category in categories:
+                if category in parsed_pipfile:
+                    parsed_pipfile[category] = self._sort_category(
+                        parsed_pipfile[category]
+                    )
+
+        # Write Pipfile once at the end
+        self.write_toml(parsed_pipfile)
+        return results
+
     def src_name_from_url(self, index_url):
         location = urllib.parse.urlsplit(index_url).netloc
         if "." in location:

pipenv/utils/requirements.py

@@ -15,7 +15,7 @@
 from pipenv.utils.pip import get_trusted_hosts
 
 
-def redact_netloc(netloc: str) -> Tuple[str]:
+def redact_netloc(netloc: str) -> str:
     """
     Replace the sensitive data in a netloc with "****", if it exists, unless it's an environment variable.
 
@@ -26,7 +26,7 @@ def redact_netloc(netloc: str) -> Tuple[str]:
     """
     netloc, (user, password) = split_auth_from_netloc(netloc)
     if user is None:
-        return (netloc,)
+        return netloc
     if password is None:
         # Check if user is an environment variable
         if not re.match(r"\$\{\w+\}", user):
@@ -42,12 +42,16 @@ def redact_netloc(netloc: str) -> Tuple[str]:
             # If it is, leave it as is
             password = ":" + password
         user = quote(user)
-    return (f"{user}{password}@{netloc}",)
+    return f"{user}{password}@{netloc}"
 
 
 def redact_auth_from_url(url: str) -> str:
     """Replace the password in a given url with ****."""
-    return _transform_url(url, redact_netloc)[0]
+
+    def _redact_netloc_wrapper(netloc: str) -> Tuple[str]:
+        return (redact_netloc(netloc),)
+
+    return _transform_url(url, _redact_netloc_wrapper)[0]
 
 
 def normalize_name(pkg) -> str:
@@ -68,9 +72,10 @@ def import_requirements(project, r=None, dev=False, categories=None):
         contents = f.read()
     if categories is None:
         categories = []
+
+    # Collect indexes and trusted hosts first
     indexes = []
     trusted_hosts = []
-    # Find and add extra indexes.
     for line in contents.split("\n"):
         index, extra_index, trusted_host, _ = parse_indexes(line.strip(), strict=True)
         if index:
@@ -79,8 +84,11 @@ def import_requirements(project, r=None, dev=False, categories=None):
             indexes.append(extra_index)
         if trusted_host:
             trusted_hosts.append(get_host_and_port(trusted_host))
-    # Convert Path object to string to avoid 'PosixPath' has no attribute 'decode' error
+
+    # Collect all packages for batch processing
+    packages_to_add = []
     req_path = str(r) if isinstance(r, Path) else r
+
     for f in parse_requirements(req_path, session=PipSession()):
         package = install_req_from_parsed_requirement(f)
         if package.name not in BAD_PACKAGES:
@@ -91,29 +99,26 @@ def import_requirements(project, r=None, dev=False, categories=None):
                     package_string = unquote(
                         redact_auth_from_url(package.original_link.url)
                     )
-
-                if categories:
-                    for category in categories:
-                        project.add_package_to_pipfile(
-                            package, package_string, dev=dev, category=category
-                        )
-                else:
-                    project.add_package_to_pipfile(package, package_string, dev=dev)
             else:
                 package_string = str(package.req)
                 if package.markers:
                     package_string += f" ; {package.markers}"
-                if categories:
-                    for category in categories:
-                        project.add_package_to_pipfile(
-                            package, package_string, dev=dev, category=category
-                        )
-                else:
-                    project.add_package_to_pipfile(package, package_string, dev=dev)
+
+            packages_to_add.append((package, package_string))
+
+    # Batch add all packages to Pipfile
+    if packages_to_add:
+        project.add_packages_to_pipfile_batch(
+            packages_to_add, dev=dev, categories=categories
+        )
+
+    # Add indexes after packages
     indexes = sorted(set(indexes))
     trusted_hosts = sorted(set(trusted_hosts))
     for index in indexes:
         add_index_to_pipfile(project, index, trusted_hosts)
+
+    # Recase pipfile once at the end
     project.recase_pipfile()