[BUG]: increase max payload size of log service (Go) (#4534)

The max gRPC payload/response sizes were updated for the Rust log
client, but were never updated for the log service. So the `PushLogs`
method was still failing when the request was > 4MB.

_How are these changes tested?_

- [x] Tests pass locally with `pytest` for python, `yarn test` for js,
`cargo test` for rust

_Are all docstrings for user-facing APIs updated if required? Do we need
to make documentation changes in the [docs
section](https://github.com/chroma-core/chroma/tree/main/docs/docs.trychroma.com)?_

n/a

checkpoint after adding default

checkpoint test passing

Author: codetheweb

clients/js/packages/chromadb-core/src/CollectionConfiguration.ts

@@ -2,7 +2,6 @@ import {
   IEmbeddingFunction,
   EmbeddingFunctionSpace,
 } from "./embeddings/IEmbeddingFunction";
-import { DefaultEmbeddingFunction } from "./embeddings/DefaultEmbeddingFunction";
 import { Api } from "./generated";
 export type HnswSpace = EmbeddingFunctionSpace;
 

clients/new-js/packages/ai-embeddings/default-embed/README.md

@@ -0,0 +1,46 @@
+{
+  "name": "@chroma-core/default-embed",
+  "version": "0.1.0",
+  "private": false,
+  "description": "Default embedding function for Chroma",
+  "main": "dist/cjs/default-embed.cjs",
+  "types": "dist/default-embed.d.ts",
+  "module": "dist/default-embed.legacy-esm.js",
+  "type": "module",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/default-embed.d.ts",
+        "default": "./dist/default-embed.mjs"
+      },
+      "require": {
+        "types": "./dist/cjs/default-embed.d.cts",
+        "default": "./dist/cjs/default-embed.cjs"
+      }
+    }
+  },
+  "files": [
+    "src",
+    "dist"
+  ],
+  "scripts": {
+    "clean": "rimraf dist",
+    "prebuild": "rimraf dist",
+    "build": "tsup",
+    "watch": "tsup --watch",
+    "test": "tsx src/test.ts"
+  },
+  "dependencies": {
+    "@huggingface/transformers": "^3.5.1"
+  },
+  "devDependencies": {
+    "rimraf": "^5.0.0",
+    "tsup": "^8.3.5"
+  },
+  "engines": {
+    "node": ">=14.17.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

clients/new-js/packages/ai-embeddings/default-embed/package.json

@@ -0,0 +1,73 @@
+import { pipeline, ProgressCallback } from "@huggingface/transformers";
+
+export type DType =
+  | "auto"
+  | "fp32"
+  | "fp16"
+  | "q8"
+  | "int8"
+  | "uint8"
+  | "q4"
+  | "bnb4"
+  | "q4f16";
+
+export type Quantization = DType | Record<string, DType>;
+
+interface StoredConfig {
+  model?: string;
+  revision?: string;
+  dtype?: Quantization;
+}
+
+export class DefaultEmbeddingFunction {
+  public readonly name: string = "default";
+  private readonly model: string;
+  private readonly revision: string;
+  private readonly dtype: Quantization | undefined;
+  private readonly progressCallback: ProgressCallback | undefined = undefined;
+
+  constructor(
+    args: Partial<
+      StoredConfig & { progressCallback: ProgressCallback | undefined }
+    > = {},
+  ) {
+    const {
+      model = "Xenova/all-MiniLM-L6-v2",
+      revision = "main",
+      dtype = undefined,
+      progressCallback = undefined,
+    } = args;
+
+    this.model = model;
+    this.revision = revision;
+    this.dtype = dtype;
+    this.progressCallback = progressCallback;
+  }
+
+  public async generate(texts: string[]): Promise<number[][]> {
+    const pipe = await pipeline("feature-extraction", this.model, {
+      revision: this.revision,
+      progress_callback: this.progressCallback,
+      dtype: this.dtype,
+    });
+
+    const output = await pipe(texts, { pooling: "mean", normalize: true });
+    return output.tolist();
+  }
+
+  public getConfig(): Record<string, any> {
+    return {
+      model: this.model,
+      revision: this.revision,
+      dtype: this.dtype,
+    };
+  }
+
+  public buildFromConfig(config: StoredConfig): DefaultEmbeddingFunction {
+    return new DefaultEmbeddingFunction(config);
+  }
+
+  static buildFromConfig(config: StoredConfig): DefaultEmbeddingFunction {
+    return new DefaultEmbeddingFunction(config);
+  }
+}

clients/new-js/packages/ai-embeddings/default-embed/src/index.ts

@@ -0,0 +1,9 @@
+import { DefaultEmbeddingFunction } from "./index";
+
+const main = async () => {
+  const e = new DefaultEmbeddingFunction();
+  const x = await e.generate(["hello"]);
+  console.log(x);
+};
+
+main().catch(console.error);

clients/new-js/packages/ai-embeddings/default-embed/src/test.ts

@@ -0,0 +1,17 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "baseUrl": "./src",
+    "stripInternal": true,
+    "composite": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "test", "**/*.test.ts"],
+  "references": [
+    {
+      "path": "../../chromadb"
+    }
+  ]
+}

clients/new-js/packages/ai-embeddings/default-embed/tsconfig.json

@@ -0,0 +1,41 @@
+import { defineConfig, Options } from "tsup";
+import * as fs from "fs";
+
+export default defineConfig((options: Options) => {
+  const commonOptions: Partial<Options> = {
+    entry: {
+      "default-embed": "src/index.ts",
+    },
+    sourcemap: true,
+    dts: {
+      compilerOptions: {
+        composite: false,
+        declaration: true,
+        emitDeclarationOnly: false,
+      },
+    },
+    ...options,
+  };
+
+  return [
+    {
+      ...commonOptions,
+      format: ["esm"],
+      outExtension: () => ({ js: ".mjs" }),
+      clean: true,
+      async onSuccess() {
+        // Support Webpack 4 by pointing `"module"` to a file with a `.js` extension
+        fs.copyFileSync(
+          "dist/default-embed.mjs",
+          "dist/default-embed.legacy-esm.js",
+        );
+      },
+    },
+    {
+      ...commonOptions,
+      format: "cjs",
+      outDir: "./dist/cjs/",
+      outExtension: () => ({ js: ".cjs" }),
+    },
+  ];
+});

clients/new-js/packages/ai-embeddings/default-embed/tsup.config.ts

@@ -29,15 +29,12 @@
     "build": "tsup",
     "watch": "tsup --watch"
   },
-  "dependencies": {
-    "chromadb": "workspace:^"
-  },
   "devDependencies": {
     "rimraf": "^5.0.0",
     "tsup": "^8.3.5"
   },
   "peerDependencies": {
-
+    "chromadb": "workspace:^"
   },
   "engines": {
     "node": ">=14.17.0"

clients/new-js/packages/ai-embeddings/jina/package.json

@@ -23,18 +23,18 @@ interface JinaRequestBody {
 }
 
 export class JinaEmbeddingFunction implements EmbeddingFunction {
-  name = "jina";
-
-  private api_key_env_var: string;
-  private model_name: string;
-  private api_url: string;
-  private headers: { [key: string]: string };
-  private task: string | undefined;
-  private late_chunking: boolean | undefined;
-  private truncate: boolean | undefined;
-  private dimensions: number | undefined;
-  private embedding_type: string | undefined;
-  private normalized: boolean | undefined;
+  public readonly name = "jina";
+
+  private readonly api_key_env_var: string;
+  private readonly model_name: string;
+  private readonly api_url: string;
+  private readonly headers: { [key: string]: string };
+  private readonly task: string | undefined;
+  private readonly late_chunking: boolean | undefined;
+  private readonly truncate: boolean | undefined;
+  private readonly dimensions: number | undefined;
+  private readonly embedding_type: string | undefined;
+  private readonly normalized: boolean | undefined;
 
   constructor({
     jinaai_api_key,
@@ -163,4 +163,4 @@ export class JinaEmbeddingFunction implements EmbeddingFunction {
   }
 }
 
-registerEmbeddingFunction(JinaEmbeddingFunction.prototype);
+registerEmbeddingFunction(JinaEmbeddingFunction);

clients/new-js/packages/ai-embeddings/jina/src/index.ts

@@ -68,7 +68,8 @@
     "tsd": "^0.28.1",
     "tsup": "^8.3.5",
     "typescript": "^5.0.4",
-    "wait-on": "^8.0.3"
+    "wait-on": "^8.0.3",
+    "@chroma-core/default-embed": "workspace:^"
   },
   "optionalDependencies": {
     "chromadb-js-bindings-darwin-arm64": "^0.1.1",

clients/new-js/packages/chromadb/package.json

@@ -9,6 +9,7 @@ import { CollectionMetadata, UserIdentity } from "./types";
 import { Collection, CollectionImpl } from "./collection";
 import {
   EmbeddingFunction,
+  getDefaultEFConfig,
   getEmbeddingFunction,
   serializeEmbeddingFunction,
 } from "./embedding-function";
@@ -126,15 +127,15 @@ export class ChromaClient {
       ...(configuration || {}),
       embedding_function: embeddingFunction
         ? serializeEmbeddingFunction(embeddingFunction)
-        : undefined,
+        : await getDefaultEFConfig(),
     };
 
     const { data } = await Api.createCollection({
       client: this.apiClient,
       path: this.path(),
       body: {
         name,
-        // configuration: undefined,
+        configuration: collectionConfig,
         metadata,
         get_or_create: false,
       },

clients/new-js/packages/chromadb/src/chroma-client.ts

@@ -16,6 +16,12 @@ export interface EmbeddingFunction {
   validateConfig?(config: Record<string, any>): void;
 }
 
+export interface EmbeddingFunctionClass {
+  new (...args: any[]): EmbeddingFunction;
+  name: string;
+  buildFromConfig(config: Record<string, any>): EmbeddingFunction;
+}
+
 class MalformedEmbeddingFunction implements EmbeddingFunction {
   public readonly name: string;
 
@@ -29,18 +35,21 @@ class MalformedEmbeddingFunction implements EmbeddingFunction {
   }
 }
 
-export const knownEmbeddingFunctions = new Map<string, EmbeddingFunction>();
+export const knownEmbeddingFunctions = new Map<
+  string,
+  EmbeddingFunctionClass
+>();
 
-export const registerEmbeddingFunction = (fn: EmbeddingFunction) => {
-  if (!fn.name) {
-    throw new Error("Embedding function must have a name to be registered.");
-  }
-  if (knownEmbeddingFunctions.has(fn.name)) {
+export const registerEmbeddingFunction = (
+  name: string,
+  fn: EmbeddingFunctionClass,
+) => {
+  if (knownEmbeddingFunctions.has(name)) {
     throw new Error(
-      `Embedding function with name ${fn.name} is already registered.`,
+      `Embedding function with name ${name} is already registered.`,
     );
   }
-  knownEmbeddingFunctions.set(fn.name, fn);
+  knownEmbeddingFunctions.set(name, fn);
 };
 
 export const getEmbeddingFunction = (
@@ -65,7 +74,7 @@ export const getEmbeddingFunction = (
   if (!embeddingFunction) {
     return new MalformedEmbeddingFunction(
       collectionName,
-      `Embedding function ${name} is not registered. Make sure that the @ai-embeddings/${name} package is installed`,
+      `Embedding function ${name} is not registered. Make sure that the @chroma-core/${name} package is installed`,
     );
   }
 
@@ -89,12 +98,8 @@ export const getEmbeddingFunction = (
 };
 
 export const serializeEmbeddingFunction = (
-  ef?: EmbeddingFunction,
+  ef: EmbeddingFunction,
 ): EmbeddingFunctionConfiguration => {
-  if (!ef) {
-    return { type: "legacy" };
-  }
-
   if (!ef.getConfig || !ef.name) {
     throw new Error(
       "Failed to serialize embedding function: missing 'getConfig' or 'name'",
@@ -108,3 +113,24 @@ export const serializeEmbeddingFunction = (
     config: ef.getConfig(),
   };
 };
+
+export const getDefaultEFConfig =
+  async (): Promise<EmbeddingFunctionConfiguration> => {
+    try {
+      const { DefaultEmbeddingFunction } = await import(
+        "@chroma-core/default-embed"
+      );
+      if (!knownEmbeddingFunctions.has(new DefaultEmbeddingFunction().name)) {
+        registerEmbeddingFunction("default", DefaultEmbeddingFunction);
+      }
+    } catch (e) {
+      throw new Error(
+        "Cannot instantiate a collection with the DefaultEmbeddingFunction. Please install @chroma-core/default-embed, or provide a different embedding function",
+      );
+    }
+    return {
+      name: "default",
+      type: "known",
+      config: {},
+    };
+  };

clients/new-js/packages/chromadb/src/collection-configuration.ts

@@ -119,7 +119,7 @@ const validateEmbeddings = ({
     );
   }
 
-  if (embeddings.filter((e) => e.every((n: any) => typeof n === "number"))) {
+  if (!embeddings.filter((e) => e.every((n: any) => typeof n === "number"))) {
     throw new Error("Expected each embedding to be an array of numbers");
   }