Skip to content

Commit d5153c3

Browse files
MohamedYasserOafHenryHengZJ
MohamedYasserOaf
and
HenryHengZJ
authored
Added meilisearch retriever component (#2824)
* added meilisearch retriever and credentials.ts * added semantic ratio * removed a TODO implementatio * meilisearch component implemented with searching and upsert functionality (#3) meilisearch retriever component created , searching for an existing index and upserting a new or existing index has been implemented , component utilizes langchain and meilisearch vector search Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/3 Reviewed-by: mohamed1999akram <[email protected]> * added CI/CD for ntg branch, added proper dockerfile for flowise-ntg (#4) Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/4 Reviewed-by: mohammad <[email protected]> * modified os version , removed linting errors , removed cypress github actions (#5) added --no-lock-file flag to pass CICD , made the runner run on debian and not ubuntu , removed code that caused warnings to pass linting Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/5 Reviewed-by: omaryassery <[email protected]> * removed unnecessary QEMU install action (#6) Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/6 Reviewed-by: omaryassery <[email protected]> * removed cypress installation and linting from dockerfile (#7) Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/7 Reviewed-by: isameh <[email protected]> * dockerfile-ntg-modification (#9) dockerfile-ntg modified to copy all working directory before calling pnpm install Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/9 Reviewed-by: isameh <[email protected]> * resolved comments, reverted CI/CD * add test docker build yml back * moved meilisearch to vector store folder * Update Meilisearch.ts --------- Co-authored-by: Henry <[email protected]> Co-authored-by: Henry Heng <[email protected]>
1 parent 0a36aa7 commit d5153c3

File tree

6 files changed

+311
-0
lines changed

6 files changed

+311
-0
lines changed

‎packages/components/credentials/MeilisearchApi.credential.ts

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import { INodeParams, INodeCredential } from '../src/Interface'
2+
3+
class MeilisearchApi implements INodeCredential {
4+
label: string
5+
name: string
6+
version: number
7+
description: string
8+
inputs: INodeParams[]
9+
10+
constructor() {
11+
this.label = 'Meilisearch API'
12+
this.name = 'meilisearchApi'
13+
this.version = 1.0
14+
this.description =
15+
'Refer to <a target="_blank" href="https://meilisearch.com">official guide</a> on how to get an API Key, you need a search API KEY for basic searching functionality, admin API KEY is optional but needed for upsert functionality '
16+
this.inputs = [
17+
{
18+
label: 'Meilisearch Search API Key',
19+
name: 'meilisearchSearchApiKey',
20+
type: 'password'
21+
},
22+
{
23+
label: 'Meilisearch Admin API Key',
24+
name: 'meilisearchAdminApiKey',
25+
type: 'password',
26+
optional: true
27+
}
28+
]
29+
}
30+
}
31+
32+
module.exports = { credClass: MeilisearchApi }

‎packages/components/nodes/vectorstores/Meilisearch/Meilisearch.png

8.64 KB
Loading

‎packages/components/nodes/vectorstores/Meilisearch/Meilisearch.ts

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
import { getCredentialData, getCredentialParam } from '../../../src'
2+
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
3+
import { Meilisearch } from 'meilisearch'
4+
import { MeilisearchRetriever } from './core'
5+
import { flatten } from 'lodash'
6+
import { Document } from '@langchain/core/documents'
7+
import { v4 as uuidv4 } from 'uuid'
8+
import { Embeddings } from '@langchain/core/embeddings'
9+
10+
class MeilisearchRetriever_node implements INode {
11+
label: string
12+
name: string
13+
version: number
14+
description: string
15+
type: string
16+
icon: string
17+
category: string
18+
baseClasses: string[]
19+
inputs: INodeParams[]
20+
credential: INodeParams
21+
badge: string
22+
outputs: INodeOutputsValue[]
23+
author?: string
24+
25+
constructor() {
26+
this.label = 'Meilisearch'
27+
this.name = 'meilisearch'
28+
this.version = 1.0
29+
this.type = 'Meilisearch'
30+
this.icon = 'Meilisearch.png'
31+
this.category = 'Vector Stores'
32+
this.badge = 'NEW'
33+
this.description = `Upsert embedded data and perform similarity search upon query using Meilisearch hybrid search functionality`
34+
this.baseClasses = ['BaseRetriever']
35+
this.credential = {
36+
label: 'Connect Credential',
37+
name: 'credential',
38+
type: 'credential',
39+
credentialNames: ['meilisearchApi']
40+
}
41+
this.inputs = [
42+
{
43+
label: 'Document',
44+
name: 'document',
45+
type: 'Document',
46+
list: true,
47+
optional: true
48+
},
49+
{
50+
label: 'Embeddings',
51+
name: 'embeddings',
52+
type: 'Embeddings'
53+
},
54+
{
55+
label: 'Host',
56+
name: 'host',
57+
type: 'string',
58+
description: 'This is the URL for the desired Meilisearch instance'
59+
},
60+
{
61+
label: 'Index Uid',
62+
name: 'indexUid',
63+
type: 'string',
64+
description: 'UID for the index to answer from'
65+
},
66+
{
67+
label: 'Top K',
68+
name: 'K',
69+
type: 'number',
70+
description: 'number of top searches to return as context',
71+
additionalParams: true,
72+
optional: true
73+
},
74+
{
75+
label: 'Semantic Ratio',
76+
name: 'semanticRatio',
77+
type: 'number',
78+
description: 'percentage of sematic reasoning in meilisearch hybrid search',
79+
additionalParams: true,
80+
optional: true
81+
}
82+
]
83+
this.outputs = [
84+
{
85+
label: 'Meilisearch Retriever',
86+
name: 'MeilisearchRetriever',
87+
description: 'retrieve answers',
88+
baseClasses: this.baseClasses
89+
}
90+
]
91+
this.outputs = [
92+
{
93+
label: 'Meilisearch Retriever',
94+
name: 'retriever',
95+
baseClasses: this.baseClasses
96+
}
97+
]
98+
}
99+
//@ts-ignore
100+
vectorStoreMethods = {
101+
async upsert(nodeData: INodeData, options: ICommonObject): Promise<any> {
102+
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
103+
const meilisearchAdminApiKey = getCredentialParam('meilisearchAdminApiKey', credentialData, nodeData)
104+
const docs = nodeData.inputs?.document as Document[]
105+
const host = nodeData.inputs?.host as string
106+
const indexUid = nodeData.inputs?.indexUid as string
107+
const embeddings = nodeData.inputs?.embeddings as Embeddings
108+
let embeddingDimension: number = 384
109+
const client = new Meilisearch({
110+
host: host,
111+
apiKey: meilisearchAdminApiKey
112+
})
113+
const flattenDocs = docs && docs.length ? flatten(docs) : []
114+
const finalDocs = []
115+
for (let i = 0; i < flattenDocs.length; i += 1) {
116+
if (flattenDocs[i] && flattenDocs[i].pageContent) {
117+
const uniqueId = uuidv4()
118+
const { pageContent, metadata } = flattenDocs[i]
119+
const docEmbedding = await embeddings.embedQuery(pageContent)
120+
embeddingDimension = docEmbedding.length
121+
const documentForIndexing = {
122+
pageContent,
123+
metadata,
124+
objectID: uniqueId,
125+
_vectors: {
126+
ollama: {
127+
embeddings: docEmbedding,
128+
regenerate: false
129+
}
130+
}
131+
}
132+
finalDocs.push(documentForIndexing)
133+
}
134+
}
135+
let index: any
136+
try {
137+
index = await client.getIndex(indexUid)
138+
} catch (error) {
139+
console.error('Error fetching index:', error)
140+
await client.createIndex(indexUid, { primaryKey: 'objectID' })
141+
} finally {
142+
index = await client.getIndex(indexUid)
143+
}
144+
145+
try {
146+
await index.updateSettings({
147+
embedders: {
148+
ollama: {
149+
source: 'userProvided',
150+
dimensions: embeddingDimension
151+
}
152+
}
153+
})
154+
await index.addDocuments(finalDocs)
155+
} catch (error) {
156+
console.error('Error occurred while adding documents:', error)
157+
}
158+
return
159+
}
160+
}
161+
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
162+
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
163+
const meilisearchSearchApiKey = getCredentialParam('meilisearchSearchApiKey', credentialData, nodeData)
164+
const host = nodeData.inputs?.host as string
165+
const indexUid = nodeData.inputs?.indexUid as string
166+
const K = nodeData.inputs?.K as string
167+
const semanticRatio = nodeData.inputs?.semanticRatio as string
168+
const embeddings = nodeData.inputs?.embeddings as Embeddings
169+
170+
const hybridsearchretriever = new MeilisearchRetriever(host, meilisearchSearchApiKey, indexUid, K, semanticRatio, embeddings)
171+
return hybridsearchretriever
172+
}
173+
}
174+
module.exports = { nodeClass: MeilisearchRetriever_node }

‎packages/components/nodes/vectorstores/Meilisearch/core.ts

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
import { BaseRetriever, type BaseRetrieverInput } from '@langchain/core/retrievers'
2+
import { Document } from '@langchain/core/documents'
3+
import { Meilisearch } from 'meilisearch'
4+
import { Embeddings } from '@langchain/core/embeddings'
5+
6+
export interface CustomRetrieverInput extends BaseRetrieverInput {}
7+
8+
export class MeilisearchRetriever extends BaseRetriever {
9+
lc_namespace = ['langchain', 'retrievers']
10+
private readonly meilisearchSearchApiKey: any
11+
private readonly host: any
12+
private indexUid: string
13+
private K: string
14+
private semanticRatio: string
15+
private embeddings: Embeddings
16+
constructor(
17+
host: string,
18+
meilisearchSearchApiKey: any,
19+
indexUid: string,
20+
K: string,
21+
semanticRatio: string,
22+
embeddings: Embeddings,
23+
fields?: CustomRetrieverInput
24+
) {
25+
super(fields)
26+
this.meilisearchSearchApiKey = meilisearchSearchApiKey
27+
this.host = host
28+
this.indexUid = indexUid
29+
this.embeddings = embeddings
30+
31+
if (semanticRatio == '') {
32+
this.semanticRatio = '0.5'
33+
} else {
34+
let semanticRatio_Float = parseFloat(semanticRatio)
35+
if (semanticRatio_Float > 1.0) {
36+
this.semanticRatio = '1.0'
37+
} else if (semanticRatio_Float < 0.0) {
38+
this.semanticRatio = '0.0'
39+
} else {
40+
this.semanticRatio = semanticRatio
41+
}
42+
}
43+
44+
if (K == '') {
45+
K = '4'
46+
}
47+
this.K = K
48+
}
49+
50+
async _getRelevantDocuments(query: string): Promise<Document[]> {
51+
// Pass `runManager?.getChild()` when invoking internal runnables to enable tracing
52+
// const additionalDocs = await someOtherRunnable.invoke(params, runManager?.getChild())
53+
const client = new Meilisearch({
54+
host: this.host,
55+
apiKey: this.meilisearchSearchApiKey
56+
})
57+
58+
const index = await client.index(this.indexUid)
59+
const questionEmbedding = await this.embeddings.embedQuery(query)
60+
// Perform the search
61+
const searchResults = await index.search(query, {
62+
vector: questionEmbedding,
63+
limit: parseInt(this.K), // Optional: Limit the number of results
64+
attributesToRetrieve: ['*'], // Optional: Specify which fields to retrieve
65+
hybrid: {
66+
semanticRatio: parseFloat(this.semanticRatio),
67+
embedder: 'ollama'
68+
}
69+
})
70+
const hits = searchResults.hits
71+
let documents: Document[] = [
72+
new Document({
73+
pageContent: 'mock page',
74+
metadata: {}
75+
})
76+
]
77+
try {
78+
documents = hits.map(
79+
(hit: any) =>
80+
new Document({
81+
pageContent: hit.pageContent,
82+
metadata: {
83+
objectID: hit.objectID
84+
}
85+
})
86+
)
87+
} catch (e) {
88+
console.error('Error occurred while adding documents:', e)
89+
}
90+
return documents
91+
}
92+
}

‎packages/components/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@
9494
"lodash": "^4.17.21",
9595
"lunary": "^0.6.16",
9696
"mammoth": "^1.5.1",
97+
"meilisearch": "^0.41.0",
9798
"moment": "^2.29.3",
9899
"mongodb": "6.3.0",
99100
"mysql2": "^3.9.2",

‎pnpm-lock.yaml

Lines changed: 12 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)