[CHORE] remove hnsw-lib from dependencies

Author: jairad26

Dockerfile

@@ -69,4 +69,3 @@ EXPOSE 8000
 
 ENTRYPOINT ["/docker_entrypoint.sh"]
 CMD [ "--workers ${CHROMA_WORKERS} --host ${CHROMA_HOST_ADDR} --port ${CHROMA_HOST_PORT} --proxy-headers --reload --log-config ${CHROMA_LOG_CONFIG} --timeout-keep-alive ${CHROMA_TIMEOUT_KEEP_ALIVE}"]
-

Dockerfile.windows

@@ -4,16 +4,25 @@ ARG REBUILD_HNSWLIB
 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 RUN [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; `Invoke-WebRequest "https://aka.ms/vs/17/release/vc_redist.x64.exe" -OutFile "vc_redist.x64.exe"; `Start-Process -filepath C:\vc_redist.x64.exe -ArgumentList "/install", "/passive", "/norestart" -Passthru | Wait-Process; `Remove-Item -Force vc_redist.x64.exe;
 
+# Install Rust
+RUN [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; `Invoke-WebRequest "https://static.rust-lang.org/rustup/dist/x86_64-pc-windows-msvc/rustup-init.exe" -OutFile "rustup-init.exe"; `Start-Process -filepath C:\rustup-init.exe -ArgumentList "-y" -Passthru | Wait-Process; `Remove-Item -Force rustup-init.exe;
+
 WORKDIR C:\\chroma
 
 COPY ./requirements.txt requirements.txt
 
 RUN pip install --no-cache-dir --upgrade -r requirements.txt
+RUN pip install --no-cache-dir maturin cffi
 RUN if ($env:REBUILD_HNSWLIB -eq 'true') { pip install --no-binary :all: --force-reinstall --no-cache-dir chroma-hnswlib }
 
 COPY ./bin/docker_entrypoint.ps1 C:\\docker_entrypoint.ps1
 COPY ./ C:\\chroma
 
+# Build and install chromadb with Rust
+RUN python -m maturin build
+RUN pip uninstall chromadb -y
+RUN pip install --prefix="C:\\install" --find-links target/wheels/ --upgrade chromadb
+
 ENV CHROMA_HOST_ADDR "0.0.0.0"
 ENV CHROMA_HOST_PORT 8000
 ENV CHROMA_WORKERS 1
@@ -23,4 +32,4 @@ ENV CHROMA_TIMEOUT_KEEP_ALIVE 30
 EXPOSE 8000
 
 ENTRYPOINT ["powershell", "C:\\\\docker_entrypoint.ps1"]
-CMD [ "--workers %CHROMA_WORKERS% --host %CHROMA_HOST_ADDR% --port %CHROMA_HOST_PORT% --proxy-headers --log-config %CHROMA_LOG_CONFIG% --timeout-keep-alive %CHROMA_TIMEOUT_KEEP_ALIVE%"]
+CMD [ "--workers %CHROMA_WORKERS% --host %CHROMA_HOST_ADDR% --port %CHROMA_HOST_PORT% --proxy-headers --reload --log-config %CHROMA_LOG_CONFIG% --timeout-keep-alive %CHROMA_TIMEOUT_KEEP_ALIVE%"]

chromadb/test/utils/cross_version.py

@@ -55,7 +55,10 @@ def install(pkg: str, path: str, dep_overrides: Dict[str, str]) -> int:
     for dep, operator_version in dep_overrides.items():
         command.append(f"{dep}{operator_version}")
 
-    command.append("--no-binary=chroma-hnswlib")
+    # Only add --no-binary=chroma-hnswlib if it's in the dependencies
+    if "chroma-hnswlib" in pkg or any("chroma-hnswlib" in dep for dep in dep_overrides):
+        command.append("--no-binary=chroma-hnswlib")
+
     command.append(f"--target={path}")
 
     print(f"Installing chromadb version {pkg} to {path}")

pyproject.toml

@@ -7,7 +7,10 @@ description = "Chroma."
 readme = "README.md"
 requires-python = ">=3.9"
 classifiers = ["Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent"]
-dependencies = ['build >= 1.0.3', 'pydantic >= 1.9', 'chroma-hnswlib==0.7.6', 'fastapi==0.115.9', 'uvicorn[standard] >= 0.18.3', 'numpy >= 1.22.5', 'posthog >= 2.4.0', 'typing_extensions >= 4.5.0', 'onnxruntime >= 1.14.1', 'opentelemetry-api>=1.2.0', 'opentelemetry-exporter-otlp-proto-grpc>=1.2.0', 'opentelemetry-instrumentation-fastapi>=0.41b0', 'opentelemetry-sdk>=1.2.0', 'tokenizers >= 0.13.2', 'pypika >= 0.48.9', 'tqdm >= 4.65.0', 'overrides >= 7.3.1', 'importlib-resources', 'graphlib_backport >= 1.0.3; python_version < "3.9"', 'grpcio >= 1.58.0', 'bcrypt >= 4.0.1', 'typer >= 0.9.0', 'kubernetes>=28.1.0', 'tenacity>=8.2.3', 'PyYAML>=6.0.0', 'mmh3>=4.0.1', 'orjson>=3.9.12', 'httpx>=0.27.0', 'rich>=10.11.0', 'jsonschema>=4.19.0']
+dependencies = ['build >= 1.0.3', 'pydantic >= 1.9', 'fastapi==0.115.9', 'uvicorn[standard] >= 0.18.3', 'numpy >= 1.22.5', 'posthog >= 2.4.0', 'typing_extensions >= 4.5.0', 'onnxruntime >= 1.14.1', 'opentelemetry-api>=1.2.0', 'opentelemetry-exporter-otlp-proto-grpc>=1.2.0', 'opentelemetry-instrumentation-fastapi>=0.41b0', 'opentelemetry-sdk>=1.2.0', 'tokenizers >= 0.13.2', 'pypika >= 0.48.9', 'tqdm >= 4.65.0', 'overrides >= 7.3.1', 'importlib-resources', 'graphlib_backport >= 1.0.3; python_version < "3.9"', 'grpcio >= 1.58.0', 'bcrypt >= 4.0.1', 'typer >= 0.9.0', 'kubernetes>=28.1.0', 'tenacity>=8.2.3', 'PyYAML>=6.0.0', 'mmh3>=4.0.1', 'orjson>=3.9.12', 'httpx>=0.27.0', 'rich>=10.11.0', 'jsonschema>=4.19.0']
+
+[project.optional-dependencies]
+dev = ['chroma-hnswlib==0.7.6']
 
 [tool.black]
 line-length = 88

requirements.txt

@@ -1,5 +1,4 @@
 bcrypt>=4.0.1
-chroma-hnswlib==0.7.6
 fastapi==0.115.9
 graphlib_backport==1.0.3; python_version < '3.9'
 grpcio>=1.58.0

requirements_dev.txt

@@ -1,5 +1,6 @@
 black==23.3.0 # match what's in pyproject.toml
 build
+chroma-hnswlib==0.7.6
 grpcio-tools==1.67.1 # Later version not compatible with protobuf 4.25.5
 httpx
 hypothesis==6.112.2 # TODO: Resolve breaking changes and bump version