More embedding models, model list pulling, and more fixes

Author: metaspartan

client/src/App.tsx

@@ -2,9 +2,58 @@ import { Switch, Route } from "wouter";
 import Dashboard from "@/pages/Dashboard";
 import { useEffect } from "react";
 import { useStore } from "@/lib/store";
+import { defaultLocalModels, modelService } from "./lib/localmodels";
+import { PRESET_ENDPOINTS } from "./lib/constants";
+import { CustomModel } from "./lib/types";
 
 function App() {
-  const { settings } = useStore();
+  const { settings, setSettings } = useStore();
+
+  useEffect(() => {
+    const fetchModels = async () => {
+      console.log('fetching models');
+      // Get all endpoints from settings
+      const endpoints = PRESET_ENDPOINTS.map(e => ({
+        url: e.url,
+        provider: 'openai'
+      }));
+  
+      // Fetch models from all configured endpoints
+      const modelPromises = endpoints.map(e => 
+        modelService.getAvailableModels(e.url)
+      );
+  
+      try {
+        const modelLists = await Promise.allSettled(modelPromises);
+        const allModels = modelLists
+          .filter((result): result is PromiseFulfilledResult<CustomModel[]> => 
+            result.status === 'fulfilled'
+          )
+          .flatMap(result => result.value);
+  
+        // Merge with defaults, removing duplicates
+        const existingIds = new Set(defaultLocalModels.map(m => m.id));
+        const newModels = allModels.filter(m => !existingIds.has(m.id));
+  
+        setSettings({
+          ...settings,
+          customModels: [
+            ...settings.customModels.filter(m => 
+              !endpoints.some(e => m.endpoint === e.url)
+            ),
+            ...defaultLocalModels,
+            ...newModels
+          ]
+        });
+      } catch (error) {
+        // console.error('Error fetching models:', error);
+      }
+    };
+  
+    fetchModels();
+    const interval = setInterval(fetchModels, 120000); // 2 minutes
+    return () => clearInterval(interval);
+  }, []);
 
   useEffect(() => {
     document.documentElement.style.setProperty('--primary-color', settings.primaryColor);

client/src/assets/vllm.svg

@@ -85,9 +85,6 @@ import { PRESET_ENDPOINTS } from "@/lib/constants";
                             />
                             <div className="flex flex-col">
                             <span>{PRESET_ENDPOINTS.find(p => p.url === newModel.endpoint)?.name}</span>
-                            <span className="text-xs text-muted-foreground">
-                                {PRESET_ENDPOINTS.find(p => p.url === newModel.endpoint)?.description}
-                            </span>
                             </div>
                         </div>
                         )}
@@ -96,17 +93,17 @@ import { PRESET_ENDPOINTS } from "@/lib/constants";
                     <SelectContent>
                     {PRESET_ENDPOINTS.map((preset) => (
                         <SelectItem key={preset.url} value={preset.url || "custom"}>
-                        <div className="flex items-center gap-2">
+                        <div className="flex items-start gap-2">
                           <img 
                             src={preset.icon} 
                             alt={preset.name} 
-                            className="w-6 h-6 object-contain"
+                            className="w-5 h-5 object-contain"
                           />
                           <div className="flex flex-col">
                             <span>{preset.name}</span>
-                            <span className="text-xs text-muted-foreground">
+                            {/* <span className="text-xs text-muted-foreground">
                               {preset.description}
-                            </span>
+                            </span> */}
                           </div>
                         </div>
                       </SelectItem>

client/src/components/AddModelDialog.tsx

@@ -32,6 +32,8 @@ import logo from "@/assets/logo.svg";
 import { RAGService } from "@/lib/rag";
 import { FileUpload } from "./FileUpload";
 import { encode } from "gpt-tokenizer";
+import { defaultLocalModels } from "@/lib/localmodels";
+import { modelService } from "@/lib/localmodels";
 
 export function ChatNode({ id, data: initialData }: NodeProps) {
   const [input, setInput] = useState("");
@@ -174,6 +176,15 @@ export function ChatNode({ id, data: initialData }: NodeProps) {
            "";
   };
 
+  const formatSource = (metadata: any) => {
+    console.log("Formatting source:", metadata);
+    console.log("Type:", metadata.type);
+    if (metadata.type === 'website') {
+      return metadata.source;
+    }
+    return metadata.filename;
+  };
+
   const sendMessage = useCallback(async () => {
     if (!input.trim()) return;
 
@@ -232,8 +243,9 @@ export function ChatNode({ id, data: initialData }: NodeProps) {
             const metadata = typeof doc.metadata === 'string' 
               ? JSON.parse(doc.metadata) 
               : doc.metadata;
-            console.log("Processing doc metadata:", metadata);
-            return `[From ${metadata.filename}]: ${doc.content}`;
+            console.log("Processing metadata:", metadata);
+            const source = formatSource(metadata);
+            return `[From ${source}]: ${doc.content}`;
           })
           .join('\n\n');
       }

client/src/components/ChatNode.tsx

@@ -7,6 +7,10 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip
 import { Info } from "lucide-react";
 import logo from "@/assets/logo.svg"
 import { PRESET_ENDPOINTS } from "@/lib/constants";
+import { modelService } from "@/lib/localmodels";
+import { defaultLocalModels } from "@/lib/localmodels";
+import { useEffect } from "react";
+
 interface ModelSelectorProps {
   models: AIModel[];
   selectedModel: string;
@@ -15,6 +19,7 @@ interface ModelSelectorProps {
 
 export function ModelSelector({ models, selectedModel, onSelect }: ModelSelectorProps) {
   const settings = useStore((state) => state.settings);
+  const setSettings = useStore((state) => state.setSettings);
   const allModels = [...models, ...(settings.customModels || [])];
 
   const getEndpointIcon = (model: AIModel | CustomModel) => {

client/src/components/ModelSelector.tsx

@@ -10,6 +10,7 @@ import { RAGService } from "@/lib/rag";
 import { Trash2, Upload } from "lucide-react";
 import { embeddingService } from "@/lib/embeddings";
 import { cn } from "@/lib/utils";
+import { toast } from "@/hooks/use-toast";
 
 interface EmbeddingProgress {
   status: 'idle' | 'loading' | 'chunking' | 'embedding' | 'complete' | 'error';
@@ -19,9 +20,27 @@ interface EmbeddingProgress {
 }
 
 const EMBEDDING_MODELS = [
-  { id: 'Xenova/bge-base-en-v1.5', name: 'BGE Base', size: '110MB' },
-  { id: 'Xenova/bge-small-en-v1.5', name: 'BGE Small', size: '50MB' },
-  { id: 'Xenova/bge-large-en-v1.5', name: 'BGE Large', size: '330MB' }
+  // HuggingFace embedding models
+  { id: 'Xenova/bge-base-en-v1.5', name: 'BGE Base English', size: '436MB', type: 'local', device: 'auto', dtype: 'fp32' },
+  { id: 'Xenova/bge-small-en-v1.5', name: 'BGE Small English', size: '133MB', type: 'local', device: 'auto', dtype: 'fp32' },
+  { id: 'Xenova/bge-large-en-v1.5', name: 'BGE Large English', size: '1.34GB', type: 'local', device: 'auto', dtype: 'fp32' },
+  { id: 'Xenova/nomic-embed-text-v1', name: 'Nomic Embed Text v1', size: '550MB', type: 'local', device: 'auto', dtype: 'fp32' },
+  { id: 'Xenova/gte-small', name: 'GTE Small', size: '133MB', type: 'local', device: 'auto', dtype: 'fp32' },
+  { id: 'Xenova/gte-large', name: 'GTE Large', size: '1.34GB', type: 'local', device: 'auto', dtype: 'fp32' },
+  { id: 'Xenova/GIST-small-Embedding-v0', name: 'GIST Small', size: '133MB', type: 'local', device: 'auto', dtype: 'fp32' },
+  { id: 'Xenova/ernie-3.0-nano-zh', name: 'ERNIE 3.0 Nano Chinese', size: '72MB', type: 'local', device: 'auto', dtype: 'fp32' },
+  { id: 'Xenova/ernie-3.0-micro-zh', name: 'ERNIE 3.0 Micro Chinese', size: '94MB', type: 'local', device: 'auto', dtype: 'fp32' },
+  { id: 'Xenova/ernie-3.0-mini-zh', name: 'ERNIE 3.0 Mini Chinese', size: '107MB', type: 'local', device: 'auto', dtype: 'fp32' },
+  { id: 'Xenova/all-roberta-large-v1', name: 'Roberta Large', size: '1.42GB', type: 'local', device: 'auto', dtype: 'fp32' },
+  { id: 'Xenova/jina-embeddings-v2-base-en', name: 'Jina Embeddings v2 Base English', size: '547MB', type: 'local', device: 'auto', dtype: 'fp32' },
+  { id: 'Xenova/jina-embeddings-v2-small-en', name: 'Jina Embeddings v2 Small English', size: '130MB', type: 'local', device: 'auto', dtype: 'fp32' },
+  { id: 'Xenova/jina-embeddings-v2-base-zh', name: 'Jina Embeddings v2 Base Chinese', size: '641MB', type: 'local', device: 'auto', dtype: 'fp32' },
+
+  // { id: 'jinaai/jina-embeddings-v3', name: 'Jina Embeddings v3', size: '2.29GB', type: 'local', device: 'auto', dtype: 'fp16' },
+
+  // OpenAI models (requires API key)
+  { id: 'text-embedding-3-small', name: 'OpenAI Embedding 3 Small', size: '', type: 'api' },
+  { id: 'text-embedding-3-large', name: 'OpenAI Embedding 3 Large', size: '', type: 'api' },
 ];
 
 export function RAGManager() {
@@ -109,9 +128,20 @@ export function RAGManager() {
       }
     }
   };
+  const isApiModel = (modelId: string) => modelId.startsWith('text-embedding-3');
 
   const handleModelChange = async (modelId: string) => {
     try {
+      // Check for API key if needed
+      if (isApiModel(modelId) && !settings.openai.apiKey) {
+        toast({
+          title: "API Key Required",
+          description: "Please configure your OpenAI API key in settings first.",
+          variant: "destructive"
+        });
+        return;
+      }
+
       // Update UI immediately
       useStore.setState({
         settings: {
@@ -252,6 +282,23 @@ export function RAGManager() {
     }
   };
 
+  const handleUrlChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    let url = e.target.value;
+    
+    // If user is typing and hasn't included a protocol, don't modify input
+    if (!url.includes('://')) {
+      setUrlInput(url);
+      return;
+    }
+    
+    // When pasting or finishing input, ensure https:// is present
+    if (!url.match(/^https?:\/\//)) {
+      url = 'https://' + url;
+    }
+    
+    setUrlInput(url);
+  };
+
   return (
     <div className="space-y-6">
                 <p className="text-sm text-muted-foreground">
@@ -320,7 +367,7 @@ export function RAGManager() {
     <SelectContent>
       {EMBEDDING_MODELS.map(model => (
         <SelectItem key={model.id} value={model.id}>
-          {model.name} ({model.size})
+          {model.name} {model.type === 'local' && `(${model.size})`} {model.type === 'api' && '(Requires API Key)'}
         </SelectItem>
       ))}
     </SelectContent>
@@ -348,13 +395,18 @@ export function RAGManager() {
           <label className="text-sm font-medium">Documents & Websites</label>
           <div className="mt-2">
           <div className="flex gap-2">
-            <Input
-              type="url"
-              placeholder="Enter website URL"
-              value={urlInput}
-              onChange={(e) => setUrlInput(e.target.value)}
-              disabled={isUploading}
-            />
+          <Input
+            type="url"
+            placeholder="Enter website URL"
+            value={urlInput}
+            onChange={handleUrlChange}
+            onBlur={() => {
+              if (urlInput && !urlInput.match(/^https?:\/\//)) {
+                setUrlInput('https://' + urlInput);
+              }
+            }}
+            disabled={isUploading}
+          />
             <Button 
               onClick={handleAddWebsite}
               disabled={isUploading || !urlInput}
@@ -363,8 +415,6 @@ export function RAGManager() {
             </Button>
           </div>
 
-        
-          
           <Input
             type="file"
             ref={inputRef}
@@ -379,18 +429,14 @@ export function RAGManager() {
               {uploadingDocs.map(docId => (
                 <div key={docId} className="flex items-center justify-between p-2 border rounded">
                   <div className="flex-1">
-                    <p className="text-sm">Uploading...</p>
+                    <p className="text-sm">Embedding...</p>
                     {progress[docId] && (
                       <div className="mt-2">
                         <Progress 
                           value={
-                            progress[docId].status === 'embedding'
-                              ? (progress[docId].currentChunk / progress[docId].totalChunks) * 100
-                              : progress[docId].status === 'loading'
-                              ? 20
-                              : progress[docId].status === 'chunking'
-                              ? 40
-                              : 0
+                       
+                             (progress[docId].currentChunk / progress[docId].totalChunks) * 100
+                              
                           }
                         />
                         <p className="text-xs text-muted-foreground mt-1">

client/src/components/RAGManager.tsx

@@ -78,7 +78,7 @@ export function SettingsDialog({ open, onOpenChange }: SettingsDialogProps) {
 
             <div className="flex justify-center">
             <img src={logo} alt="Curiso.ai" title="Curiso.ai" className="w-12 h-12" /></div>
-            <div className="flex justify-center"><p className="text-sm text-muted-foreground justify-center mb-2">Version v1.0.9 by Carsen Klock</p></div>
+            <div className="flex justify-center"><p className="text-sm text-muted-foreground justify-center mb-2">Version v1.1.0 by Carsen Klock</p></div>
             <strong>Curiso.ai</strong> is an infinite canvas for your thoughts—a platform that seamlessly connects nodes and AI services so you can explore ideas in depth without repeating yourself. By guiding the direction of each conversation, Curiso.ai empowers advanced users to unlock richer, more accurate AI interactions.
             </div>
             <div className="space-y-2 mt-2">

client/src/components/SettingsDialog.tsx

@@ -2,6 +2,8 @@ import ollamaLogo from "@/assets/ollama-logo.svg";
 import exoLogo from "@/assets/exo.png";
 import janLogo from "@/assets/jan.svg";
 import lmstudioLogo from "@/assets/lmstudio.png";
+import vllmlogo from "@/assets/vllm.svg";
+import logo from "@/assets/logo.svg";
 
 export const themeColors = [
     // Blues & Teals
@@ -64,10 +66,16 @@ export const DEFAULT_AI_SETTINGS = {
       description: "Local LM Studio instance",
       icon: lmstudioLogo
     },
+    {
+      name: "vLLM",
+      url: "http://localhost:8000/v1",
+      description: "Local vLLM instance",
+      icon: vllmlogo
+    },
     {
       name: "Custom",
       url: "",
       description: "Custom endpoint URL",
-      icon: "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='white' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Ccircle cx='12' cy='12' r='10'%3E%3C/circle%3E%3Cpath d='M12 8v8'%3E%3C/path%3E%3Cpath d='M8 12h8'%3E%3C/path%3E%3C/svg%3E"
+      icon: logo
     }
   ] as const;