Skip to content

feature/apiloader-ca-upload #4533

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 5, 2025
Merged

feature/apiloader-ca-upload #4533

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 56 additions & 14 deletions packages/components/nodes/documentloaders/API/APILoader.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import axios, { AxiosRequestConfig } from 'axios'
import { omit } from 'lodash'
import { Document } from '@langchain/core/documents'
import { TextSplitter } from 'langchain/text_splitter'
import axios, { AxiosRequestConfig } from 'axios'
import * as https from 'https'
import { BaseDocumentLoader } from 'langchain/document_loaders/base'
import { TextSplitter } from 'langchain/text_splitter'
import { omit } from 'lodash'
import { getFileFromStorage } from '../../../src'
import { ICommonObject, IDocument, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { handleEscapeCharacters } from '../../../src/utils'

Expand All @@ -21,7 +23,7 @@ class API_DocumentLoaders implements INode {
constructor() {
this.label = 'API Loader'
this.name = 'apiLoader'
this.version = 2.0
this.version = 2.1
this.type = 'Document'
this.icon = 'api.svg'
this.category = 'Document Loaders'
Expand Down Expand Up @@ -61,6 +63,15 @@ class API_DocumentLoaders implements INode {
additionalParams: true,
optional: true
},
{
label: 'SSL Certificate',
description: 'Please upload a SSL certificate file in either .pem or .crt',
name: 'caFile',
type: 'file',
fileType: '.pem, .crt',
additionalParams: true,
optional: true
},
{
label: 'Body',
name: 'body',
Expand Down Expand Up @@ -105,8 +116,10 @@ class API_DocumentLoaders implements INode {
}
]
}
async init(nodeData: INodeData): Promise<any> {

async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const headers = nodeData.inputs?.headers as string
const caFileBase64 = nodeData.inputs?.caFile as string
const url = nodeData.inputs?.url as string
const body = nodeData.inputs?.body as string
const method = nodeData.inputs?.method as string
Expand All @@ -120,22 +133,37 @@ class API_DocumentLoaders implements INode {
omitMetadataKeys = _omitMetadataKeys.split(',').map((key) => key.trim())
}

const options: ApiLoaderParams = {
const apiLoaderParam: ApiLoaderParams = {
url,
method
}

if (headers) {
const parsedHeaders = typeof headers === 'object' ? headers : JSON.parse(headers)
options.headers = parsedHeaders
apiLoaderParam.headers = parsedHeaders
}

if (caFileBase64.startsWith('FILE-STORAGE::')) {
let file = caFileBase64.replace('FILE-STORAGE::', '')
file = file.replace('[', '')
file = file.replace(']', '')
const orgId = options.orgId
const chatflowid = options.chatflowid
const fileData = await getFileFromStorage(file, orgId, chatflowid)
apiLoaderParam.ca = fileData.toString()
} else {
const splitDataURI = caFileBase64.split(',')
splitDataURI.pop()
const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
apiLoaderParam.ca = bf.toString('utf-8')
}

if (body) {
const parsedBody = typeof body === 'object' ? body : JSON.parse(body)
options.body = parsedBody
apiLoaderParam.body = parsedBody
}

const loader = new ApiLoader(options)
const loader = new ApiLoader(apiLoaderParam)

let docs: IDocument[] = []

Expand Down Expand Up @@ -195,6 +223,7 @@ interface ApiLoaderParams {
method: string
headers?: ICommonObject
body?: ICommonObject
ca?: string
}

class ApiLoader extends BaseDocumentLoader {
Expand All @@ -206,28 +235,36 @@ class ApiLoader extends BaseDocumentLoader {

public readonly method: string

constructor({ url, headers, body, method }: ApiLoaderParams) {
public readonly ca?: string

constructor({ url, headers, body, method, ca }: ApiLoaderParams) {
super()
this.url = url
this.headers = headers
this.body = body
this.method = method
this.ca = ca
}

public async load(): Promise<IDocument[]> {
if (this.method === 'POST') {
return this.executePostRequest(this.url, this.headers, this.body)
return this.executePostRequest(this.url, this.headers, this.body, this.ca)
} else {
return this.executeGetRequest(this.url, this.headers)
return this.executeGetRequest(this.url, this.headers, this.ca)
}
}

protected async executeGetRequest(url: string, headers?: ICommonObject): Promise<IDocument[]> {
protected async executeGetRequest(url: string, headers?: ICommonObject, ca?: string): Promise<IDocument[]> {
try {
const config: AxiosRequestConfig = {}
if (headers) {
config.headers = headers
}
if (ca) {
config.httpsAgent = new https.Agent({
ca: ca
})
}
const response = await axios.get(url, config)
const responseJsonString = JSON.stringify(response.data, null, 2)
const doc = new Document({
Expand All @@ -242,12 +279,17 @@ class ApiLoader extends BaseDocumentLoader {
}
}

protected async executePostRequest(url: string, headers?: ICommonObject, body?: ICommonObject): Promise<IDocument[]> {
protected async executePostRequest(url: string, headers?: ICommonObject, body?: ICommonObject, ca?: string): Promise<IDocument[]> {
try {
const config: AxiosRequestConfig = {}
if (headers) {
config.headers = headers
}
if (ca) {
config.httpsAgent = new https.Agent({
ca: ca
})
}
const response = await axios.post(url, body ?? {}, config)
const responseJsonString = JSON.stringify(response.data, null, 2)
const doc = new Document({
Expand Down