Skip to content

Commit b02f279

Browse files
HenryHengZJvinodkiran
HenryHengZJ
and
vinodkiran
authored
Feature/Code Interpreter (#3183)
* Base changes for ServerSide Events (instead of socket.io) * lint fixes * adding of interface and separate methods for streaming events * lint * first draft, handles both internal and external prediction end points. * lint fixes * additional internal end point for streaming and associated changes * return streamresponse as true to build agent flow * 1) JSON formatting for internal events 2) other fixes * 1) convert internal event to metadata to maintain consistency with external response * fix action and metadata streaming * fix for error when agent flow is aborted * prevent subflows from streaming and other code cleanup * prevent streaming from enclosed tools * add fix for preventing chaintool streaming * update lock file * add open when hidden to sse * Streaming errors * Streaming errors * add fix for showing error message * add code interpreter * add artifacts to view message dialog * Update pnpm-lock.yaml --------- Co-authored-by: Vinod Paidimarry <[email protected]>
1 parent 26444ac commit b02f279

File tree

21 files changed

+729
-333
lines changed

21 files changed

+729
-333
lines changed

‎packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,7 @@ class OpenAIAssistant_Agents implements INode {
208208

209209
const usedTools: IUsedTool[] = []
210210
const fileAnnotations = []
211+
const artifacts = []
211212

212213
const assistant = await appDataSource.getRepository(databaseEntities['Assistant']).findOneBy({
213214
id: selectedAssistantId
@@ -439,21 +440,23 @@ class OpenAIAssistant_Agents implements INode {
439440
const fileId = chunk.image_file.file_id
440441
const fileObj = await openai.files.retrieve(fileId)
441442

442-
const buffer = await downloadImg(openai, fileId, `${fileObj.filename}.png`, options.chatflowid, options.chatId)
443-
const base64String = Buffer.from(buffer).toString('base64')
444-
445-
// TODO: Use a file path and retrieve image on the fly. Storing as base64 to localStorage and database will easily hit limits
446-
const imgHTML = `<img src="data:image/png;base64,${base64String}" width="100%" height="max-content" alt="${fileObj.filename}" /><br/>`
447-
text += imgHTML
443+
const filePath = await downloadImg(
444+
openai,
445+
fileId,
446+
`${fileObj.filename}.png`,
447+
options.chatflowid,
448+
options.chatId
449+
)
450+
artifacts.push({ type: 'png', data: filePath })
448451

449452
if (!isStreamingStarted) {
450453
isStreamingStarted = true
451454
if (sseStreamer) {
452-
sseStreamer.streamStartEvent(chatId, imgHTML)
455+
sseStreamer.streamStartEvent(chatId, ' ')
453456
}
454457
}
455458
if (sseStreamer) {
456-
sseStreamer.streamTokenEvent(chatId, imgHTML)
459+
sseStreamer.streamArtifactsEvent(chatId, artifacts)
457460
}
458461
}
459462
}
@@ -565,6 +568,7 @@ class OpenAIAssistant_Agents implements INode {
565568
return {
566569
text,
567570
usedTools,
571+
artifacts,
568572
fileAnnotations,
569573
assistant: { assistantId: openAIAssistantId, threadId, runId: runThreadId, messages: messageData }
570574
}
@@ -769,12 +773,8 @@ class OpenAIAssistant_Agents implements INode {
769773
const fileId = content.image_file.file_id
770774
const fileObj = await openai.files.retrieve(fileId)
771775

772-
const buffer = await downloadImg(openai, fileId, `${fileObj.filename}.png`, options.chatflowid, options.chatId)
773-
const base64String = Buffer.from(buffer).toString('base64')
774-
775-
// TODO: Use a file path and retrieve image on the fly. Storing as base64 to localStorage and database will easily hit limits
776-
const imgHTML = `<img src="data:image/png;base64,${base64String}" width="100%" height="max-content" alt="${fileObj.filename}" /><br/>`
777-
returnVal += imgHTML
776+
const filePath = await downloadImg(openai, fileId, `${fileObj.filename}.png`, options.chatflowid, options.chatId)
777+
artifacts.push({ type: 'png', data: filePath })
778778
}
779779
}
780780

@@ -787,6 +787,7 @@ class OpenAIAssistant_Agents implements INode {
787787
return {
788788
text: returnVal,
789789
usedTools,
790+
artifacts,
790791
fileAnnotations,
791792
assistant: { assistantId: openAIAssistantId, threadId, runId: runThreadId, messages: messageData }
792793
}
@@ -807,9 +808,9 @@ const downloadImg = async (openai: OpenAI, fileId: string, fileName: string, ...
807808
const image_data_buffer = Buffer.from(image_data)
808809
const mime = 'image/png'
809810

810-
await addSingleFileToStorage(mime, image_data_buffer, fileName, ...paths)
811+
const res = await addSingleFileToStorage(mime, image_data_buffer, fileName, ...paths)
811812

812-
return image_data_buffer
813+
return res
813814
}
814815

815816
const downloadFile = async (openAIApiKey: string, fileObj: any, fileName: string, ...paths: string[]) => {

‎packages/components/nodes/agents/ToolAgent/ToolAgent.ts

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ class ToolAgent_Agents implements INode {
134134
let res: ChainValues = {}
135135
let sourceDocuments: ICommonObject[] = []
136136
let usedTools: IUsedTool[] = []
137+
let artifacts = []
137138

138139
if (shouldStreamResponse) {
139140
const handler = new CustomChainHandler(sseStreamer, chatId)
@@ -150,6 +151,12 @@ class ToolAgent_Agents implements INode {
150151
}
151152
usedTools = res.usedTools
152153
}
154+
if (res.artifacts) {
155+
if (sseStreamer) {
156+
sseStreamer.streamArtifactsEvent(chatId, flatten(res.artifacts))
157+
}
158+
artifacts = res.artifacts
159+
}
153160
// If the tool is set to returnDirect, stream the output to the client
154161
if (res.usedTools && res.usedTools.length) {
155162
let inputTools = nodeData.inputs?.tools
@@ -169,6 +176,9 @@ class ToolAgent_Agents implements INode {
169176
if (res.usedTools) {
170177
usedTools = res.usedTools
171178
}
179+
if (res.artifacts) {
180+
artifacts = res.artifacts
181+
}
172182
}
173183

174184
let output = res?.output
@@ -203,14 +213,17 @@ class ToolAgent_Agents implements INode {
203213

204214
let finalRes = output
205215

206-
if (sourceDocuments.length || usedTools.length) {
216+
if (sourceDocuments.length || usedTools.length || artifacts.length) {
207217
const finalRes: ICommonObject = { text: output }
208218
if (sourceDocuments.length) {
209219
finalRes.sourceDocuments = flatten(sourceDocuments)
210220
}
211221
if (usedTools.length) {
212222
finalRes.usedTools = usedTools
213223
}
224+
if (artifacts.length) {
225+
finalRes.artifacts = artifacts
226+
}
214227
return finalRes
215228
}
216229

‎packages/components/nodes/sequentialagents/Agent/Agent.ts

Lines changed: 34 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import {
2121
IDocument,
2222
IStateWithMessages
2323
} from '../../../src/Interface'
24-
import { ToolCallingAgentOutputParser, AgentExecutor, SOURCE_DOCUMENTS_PREFIX } from '../../../src/agents'
24+
import { ToolCallingAgentOutputParser, AgentExecutor, SOURCE_DOCUMENTS_PREFIX, ARTIFACTS_PREFIX } from '../../../src/agents'
2525
import { getInputVariables, getVars, handleEscapeCharacters, prepareSandboxVars } from '../../../src/utils'
2626
import {
2727
customGet,
@@ -35,7 +35,6 @@ import {
3535
} from '../commonUtils'
3636
import { END, StateGraph } from '@langchain/langgraph'
3737
import { StructuredTool } from '@langchain/core/tools'
38-
import { DynamicStructuredTool } from '../../tools/CustomTool/core'
3938

4039
const defaultApprovalPrompt = `You are about to execute tool: {tools}. Ask if user want to proceed`
4140
const examplePrompt = 'You are a research assistant who can search for up-to-date info using search engine.'
@@ -739,14 +738,22 @@ async function agentNode(
739738

740739
// If the last message is a tool message and is an interrupted message, format output into standard agent output
741740
if (lastMessage._getType() === 'tool' && lastMessage.additional_kwargs?.nodeId === nodeData.id) {
742-
let formattedAgentResult: { output?: string; usedTools?: IUsedTool[]; sourceDocuments?: IDocument[] } = {}
741+
let formattedAgentResult: {
742+
output?: string
743+
usedTools?: IUsedTool[]
744+
sourceDocuments?: IDocument[]
745+
artifacts?: ICommonObject[]
746+
} = {}
743747
formattedAgentResult.output = result.content
744748
if (lastMessage.additional_kwargs?.usedTools) {
745749
formattedAgentResult.usedTools = lastMessage.additional_kwargs.usedTools as IUsedTool[]
746750
}
747751
if (lastMessage.additional_kwargs?.sourceDocuments) {
748752
formattedAgentResult.sourceDocuments = lastMessage.additional_kwargs.sourceDocuments as IDocument[]
749753
}
754+
if (lastMessage.additional_kwargs?.artifacts) {
755+
formattedAgentResult.artifacts = lastMessage.additional_kwargs.artifacts as ICommonObject[]
756+
}
750757
result = formattedAgentResult
751758
} else {
752759
result.name = name
@@ -765,12 +772,18 @@ async function agentNode(
765772
if (result.sourceDocuments) {
766773
additional_kwargs.sourceDocuments = result.sourceDocuments
767774
}
775+
if (result.artifacts) {
776+
additional_kwargs.artifacts = result.artifacts
777+
}
768778
if (result.output) {
769779
result.content = result.output
770780
delete result.output
771781
}
772782

773-
const outputContent = typeof result === 'string' ? result : result.content || result.output
783+
let outputContent = typeof result === 'string' ? result : result.content || result.output
784+
785+
// remove invalid markdown image pattern: ![<some-string>](<some-string>)
786+
outputContent = typeof outputContent === 'string' ? outputContent.replace(/!\[.*?\]\(.*?\)/g, '') : outputContent
774787

775788
if (nodeData.inputs?.updateStateMemoryUI || nodeData.inputs?.updateStateMemoryCode) {
776789
let formattedOutput = {
@@ -931,6 +944,9 @@ class ToolNode<T extends BaseMessage[] | MessagesState> extends RunnableCallable
931944
// Extract all properties except messages for IStateWithMessages
932945
const { messages: _, ...inputWithoutMessages } = Array.isArray(input) ? { messages: input } : input
933946
const ChannelsWithoutMessages = {
947+
chatId: this.options.chatId,
948+
sessionId: this.options.sessionId,
949+
input: this.inputQuery,
934950
state: inputWithoutMessages
935951
}
936952

@@ -940,12 +956,14 @@ class ToolNode<T extends BaseMessage[] | MessagesState> extends RunnableCallable
940956
if (tool === undefined) {
941957
throw new Error(`Tool ${call.name} not found.`)
942958
}
943-
if (tool && tool instanceof DynamicStructuredTool) {
959+
if (tool && (tool as any).setFlowObject) {
944960
// @ts-ignore
945961
tool.setFlowObject(ChannelsWithoutMessages)
946962
}
947963
let output = await tool.invoke(call.args, config)
948964
let sourceDocuments: Document[] = []
965+
let artifacts = []
966+
949967
if (output?.includes(SOURCE_DOCUMENTS_PREFIX)) {
950968
const outputArray = output.split(SOURCE_DOCUMENTS_PREFIX)
951969
output = outputArray[0]
@@ -956,12 +974,23 @@ class ToolNode<T extends BaseMessage[] | MessagesState> extends RunnableCallable
956974
console.error('Error parsing source documents from tool')
957975
}
958976
}
977+
if (output?.includes(ARTIFACTS_PREFIX)) {
978+
const outputArray = output.split(ARTIFACTS_PREFIX)
979+
output = outputArray[0]
980+
try {
981+
artifacts = JSON.parse(outputArray[1])
982+
} catch (e) {
983+
console.error('Error parsing artifacts from tool')
984+
}
985+
}
986+
959987
return new ToolMessage({
960988
name: tool.name,
961989
content: typeof output === 'string' ? output : JSON.stringify(output),
962990
tool_call_id: call.id!,
963991
additional_kwargs: {
964992
sourceDocuments,
993+
artifacts,
965994
args: call.args,
966995
usedTools: [
967996
{

‎packages/components/nodes/sequentialagents/ToolNode/ToolNode.ts

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,12 @@ import {
1212
import { AIMessage, AIMessageChunk, BaseMessage, ToolMessage } from '@langchain/core/messages'
1313
import { StructuredTool } from '@langchain/core/tools'
1414
import { RunnableConfig } from '@langchain/core/runnables'
15-
import { SOURCE_DOCUMENTS_PREFIX } from '../../../src/agents'
15+
import { ARTIFACTS_PREFIX, SOURCE_DOCUMENTS_PREFIX } from '../../../src/agents'
1616
import { Document } from '@langchain/core/documents'
1717
import { DataSource } from 'typeorm'
1818
import { MessagesState, RunnableCallable, customGet, getVM } from '../commonUtils'
1919
import { getVars, prepareSandboxVars } from '../../../src/utils'
2020
import { ChatPromptTemplate } from '@langchain/core/prompts'
21-
import { DynamicStructuredTool } from '../../tools/CustomTool/core'
2221

2322
const defaultApprovalPrompt = `You are about to execute tool: {tools}. Ask if user want to proceed`
2423

@@ -408,6 +407,9 @@ class ToolNode<T extends IStateWithMessages | BaseMessage[] | MessagesState> ext
408407
// Extract all properties except messages for IStateWithMessages
409408
const { messages: _, ...inputWithoutMessages } = Array.isArray(input) ? { messages: input } : input
410409
const ChannelsWithoutMessages = {
410+
chatId: this.options.chatId,
411+
sessionId: this.options.sessionId,
412+
input: this.inputQuery,
411413
state: inputWithoutMessages
412414
}
413415

@@ -417,12 +419,13 @@ class ToolNode<T extends IStateWithMessages | BaseMessage[] | MessagesState> ext
417419
if (tool === undefined) {
418420
throw new Error(`Tool ${call.name} not found.`)
419421
}
420-
if (tool && tool instanceof DynamicStructuredTool) {
422+
if (tool && (tool as any).setFlowObject) {
421423
// @ts-ignore
422424
tool.setFlowObject(ChannelsWithoutMessages)
423425
}
424426
let output = await tool.invoke(call.args, config)
425427
let sourceDocuments: Document[] = []
428+
let artifacts = []
426429
if (output?.includes(SOURCE_DOCUMENTS_PREFIX)) {
427430
const outputArray = output.split(SOURCE_DOCUMENTS_PREFIX)
428431
output = outputArray[0]
@@ -433,12 +436,23 @@ class ToolNode<T extends IStateWithMessages | BaseMessage[] | MessagesState> ext
433436
console.error('Error parsing source documents from tool')
434437
}
435438
}
439+
if (output?.includes(ARTIFACTS_PREFIX)) {
440+
const outputArray = output.split(ARTIFACTS_PREFIX)
441+
output = outputArray[0]
442+
try {
443+
artifacts = JSON.parse(outputArray[1])
444+
} catch (e) {
445+
console.error('Error parsing artifacts from tool')
446+
}
447+
}
448+
436449
return new ToolMessage({
437450
name: tool.name,
438451
content: typeof output === 'string' ? output : JSON.stringify(output),
439452
tool_call_id: call.id!,
440453
additional_kwargs: {
441454
sourceDocuments,
455+
artifacts,
442456
args: call.args,
443457
usedTools: [
444458
{
@@ -489,7 +503,8 @@ const getReturnOutput = async (
489503
tool: output.name,
490504
toolInput: output.additional_kwargs.args,
491505
toolOutput: output.content,
492-
sourceDocuments: output.additional_kwargs.sourceDocuments
506+
sourceDocuments: output.additional_kwargs.sourceDocuments,
507+
artifacts: output.additional_kwargs.artifacts
493508
} as IUsedTool
494509
})
495510

0 commit comments

Comments
 (0)