feat(Chat Trigger Node): Add support for file uploads & harmonize public and development chat (#9802)

Signed-off-by: Oleg Ivaniv <me@olegivaniv.com>

packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ToolsAgent/description.ts

@@ -38,6 +38,14 @@ export const toolsAgentProperties: INodeProperties[] = [
 				default: false,
 				description: 'Whether or not the output should include intermediate steps the agent took',
 			},
+			{
+				displayName: 'Automatically Passthrough Binary Images',
+				name: 'passthroughBinaryImages',
+				type: 'boolean',
+				default: true,
+				description:
+					'Whether or not binary images should be automatically passed through to the agent as image type messages',
+			},
 		],
 	},
 ];

packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ToolsAgent/execute.ts

@@ -1,9 +1,10 @@
-import { NodeConnectionType, NodeOperationError } from 'n8n-workflow';
+import { BINARY_ENCODING, NodeConnectionType, NodeOperationError } from 'n8n-workflow';
 import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow';
 
 import type { AgentAction, AgentFinish, AgentStep } from 'langchain/agents';
 import { AgentExecutor, createToolCallingAgent } from 'langchain/agents';
 import type { BaseChatMemory } from '@langchain/community/memory/chat_memory';
+import type { BaseMessagePromptTemplateLike } from '@langchain/core/prompts';
 import { ChatPromptTemplate } from '@langchain/core/prompts';
 import { omit } from 'lodash';
 import type { Tool } from '@langchain/core/tools';
@@ -13,6 +14,7 @@ import type { ZodObject } from 'zod';
 import { z } from 'zod';
 import type { BaseOutputParser, StructuredOutputParser } from '@langchain/core/output_parsers';
 import { OutputFixingParser } from 'langchain/output_parsers';
+import { HumanMessage } from '@langchain/core/messages';
 import {
 	isChatInstance,
 	getPromptInputByType,
@@ -39,6 +41,40 @@ function getOutputParserSchema(outputParser: BaseOutputParser): ZodObject<any, a
 	return schema;
 }
 
+async function extractBinaryMessages(ctx: IExecuteFunctions) {
+	const binaryData = ctx.getInputData(0, 'main')?.[0]?.binary ?? {};
+	const binaryMessages = await Promise.all(
+		Object.values(binaryData)
+			.filter((data) => data.mimeType.startsWith('image/'))
+			.map(async (data) => {
+				let binaryUrlString;
+
+				// In filesystem mode we need to get binary stream by id before converting it to buffer
+				if (data.id) {
+					const binaryBuffer = await ctx.helpers.binaryToBuffer(
+						await ctx.helpers.getBinaryStream(data.id),
+					);
+
+					binaryUrlString = `data:${data.mimeType};base64,${Buffer.from(binaryBuffer).toString(BINARY_ENCODING)}`;
+				} else {
+					binaryUrlString = data.data.includes('base64')
+						? data.data
+						: `data:${data.mimeType};base64,${data.data}`;
+				}
+
+				return {
+					type: 'image_url',
+					image_url: {
+						url: binaryUrlString,
+					},
+				};
+			}),
+	);
+	return new HumanMessage({
+		content: [...binaryMessages],
+	});
+}
+
 export async function toolsAgentExecute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
 	this.logger.verbose('Executing Tools Agent');
 	const model = await this.getInputConnectionData(NodeConnectionType.AiLanguageModel, 0);
@@ -113,12 +149,20 @@ export async function toolsAgentExecute(this: IExecuteFunctions): Promise<INodeE
 		returnIntermediateSteps?: boolean;
 	};
 
-	const prompt = ChatPromptTemplate.fromMessages([
+	const passthroughBinaryImages = this.getNodeParameter('options.passthroughBinaryImages', 0, true);
+	const messages: BaseMessagePromptTemplateLike[] = [
 		['system', `{system_message}${outputParser ? '\n\n{formatting_instructions}' : ''}`],
 		['placeholder', '{chat_history}'],
 		['human', '{input}'],
 		['placeholder', '{agent_scratchpad}'],
-	]);
+	];
+
+	const hasBinaryData = this.getInputData(0, 'main')?.[0]?.binary !== undefined;
+	if (hasBinaryData && passthroughBinaryImages) {
+		const binaryMessage = await extractBinaryMessages(this);
+		messages.push(binaryMessage);
+	}
+	const prompt = ChatPromptTemplate.fromMessages(messages);
 
 	const agent = createToolCallingAgent({
 		llm: model,

packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentDefaultDataLoader/DocumentDefaultDataLoader.node.ts

@@ -109,6 +109,30 @@ export class DocumentDefaultDataLoader implements INodeType {
 					},
 				],
 			},
+			{
+				displayName: 'Mode',
+				name: 'binaryMode',
+				type: 'options',
+				default: 'allInputData',
+				required: true,
+				displayOptions: {
+					show: {
+						dataType: ['binary'],
+					},
+				},
+				options: [
+					{
+						name: 'Load All Input Data',
+						value: 'allInputData',
+						description: 'Use all Binary data that flows into the parent agent or chain',
+					},
+					{
+						name: 'Load Specific Data',
+						value: 'specificField',
+						description: 'Load data from a specific field in the parent agent or chain',
+					},
+				],
+			},
 			{
 				displayName: 'Data Format',
 				name: 'loader',
@@ -187,6 +211,9 @@ export class DocumentDefaultDataLoader implements INodeType {
 					show: {
 						dataType: ['binary'],
 					},
+					hide: {
+						binaryMode: ['allInputData'],
+					},
 				},
 			},
 			{

packages/@n8n/nodes-langchain/nodes/trigger/ChatTrigger/ChatTrigger.node.ts

@@ -1,11 +1,15 @@
-import {
-	type IDataObject,
-	type IWebhookFunctions,
-	type IWebhookResponseData,
-	type INodeType,
-	type INodeTypeDescription,
-	NodeConnectionType,
+import { Node, NodeConnectionType } from 'n8n-workflow';
+import type {
+	IDataObject,
+	IWebhookFunctions,
+	IWebhookResponseData,
+	INodeTypeDescription,
+	MultiPartFormData,
+	INodeExecutionData,
+	IBinaryData,
+	INodeProperties,
 } from 'n8n-workflow';
+
 import { pick } from 'lodash';
 import type { BaseChatMemory } from '@langchain/community/memory/chat_memory';
 import { createPage } from './templates';
@@ -13,15 +17,31 @@ import { validateAuth } from './GenericFunctions';
 import type { LoadPreviousSessionChatOption } from './types';
 
 const CHAT_TRIGGER_PATH_IDENTIFIER = 'chat';
+const allowFileUploadsOption: INodeProperties = {
+	displayName: 'Allow File Uploads',
+	name: 'allowFileUploads',
+	type: 'boolean',
+	default: false,
+	description: 'Whether to allow file uploads in the chat',
+};
+const allowedFileMimeTypeOption: INodeProperties = {
+	displayName: 'Allowed File Mime Types',
+	name: 'allowedFilesMimeTypes',
+	type: 'string',
+	default: '*',
+	placeholder: 'e.g. image/*, text/*, application/pdf',
+	description:
+		'Allowed file types for upload. Comma-separated list of <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types" target="_blank">MIME types</a>.',
+};
 
-export class ChatTrigger implements INodeType {
+export class ChatTrigger extends Node {
 	description: INodeTypeDescription = {
 		displayName: 'Chat Trigger',
 		name: 'chatTrigger',
 		icon: 'fa:comments',
 		iconColor: 'black',
 		group: ['trigger'],
-		version: 1,
+		version: [1, 1.1],
 		description: 'Runs the workflow when an n8n generated webchat is submitted',
 		defaults: {
 			name: 'When chat message received',
@@ -194,6 +214,20 @@ export class ChatTrigger implements INodeType {
 				default: 'Hi there! 👋\nMy name is Nathan. How can I assist you today?',
 				description: 'Default messages shown at the start of the chat, one per line',
 			},
+			{
+				displayName: 'Options',
+				name: 'options',
+				type: 'collection',
+				displayOptions: {
+					show: {
+						public: [false],
+						'@version': [{ _cnd: { gte: 1.1 } }],
+					},
+				},
+				placeholder: 'Add Field',
+				default: {},
+				options: [allowFileUploadsOption, allowedFileMimeTypeOption],
+			},
 			{
 				displayName: 'Options',
 				name: 'options',
@@ -207,6 +241,22 @@ export class ChatTrigger implements INodeType {
 				placeholder: 'Add Field',
 				default: {},
 				options: [
+					{
+						...allowFileUploadsOption,
+						displayOptions: {
+							show: {
+								'/mode': ['hostedChat'],
+							},
+						},
+					},
+					{
+						...allowedFileMimeTypeOption,
+						displayOptions: {
+							show: {
+								'/mode': ['hostedChat'],
+							},
+						},
+					},
 					{
 						displayName: 'Input Placeholder',
 						name: 'inputPlaceholder',
@@ -320,11 +370,73 @@ export class ChatTrigger implements INodeType {
 		],
 	};
 
-	async webhook(this: IWebhookFunctions): Promise<IWebhookResponseData> {
-		const res = this.getResponseObject();
+	private async handleFormData(context: IWebhookFunctions) {
+		const req = context.getRequestObject() as MultiPartFormData.Request;
+		const options = context.getNodeParameter('options', {}) as IDataObject;
+		const { data, files } = req.body;
 
-		const isPublic = this.getNodeParameter('public', false) as boolean;
-		const nodeMode = this.getNodeParameter('mode', 'hostedChat') as string;
+		const returnItem: INodeExecutionData = {
+			json: data,
+		};
+
+		if (files && Object.keys(files).length) {
+			returnItem.json.files = [] as Array<Omit<IBinaryData, 'data'>>;
+			returnItem.binary = {};
+
+			const count = 0;
+			for (const fileKey of Object.keys(files)) {
+				const processedFiles: MultiPartFormData.File[] = [];
+				if (Array.isArray(files[fileKey])) {
+					processedFiles.push(...files[fileKey]);
+				} else {
+					processedFiles.push(files[fileKey]);
+				}
+
+				let fileIndex = 0;
+				for (const file of processedFiles) {
+					let binaryPropertyName = 'data';
+
+					// Remove the '[]' suffix from the binaryPropertyName if it exists
+					if (binaryPropertyName.endsWith('[]')) {
+						binaryPropertyName = binaryPropertyName.slice(0, -2);
+					}
+					if (options.binaryPropertyName) {
+						binaryPropertyName = `${options.binaryPropertyName.toString()}${count}`;
+					}
+
+					const binaryFile = await context.nodeHelpers.copyBinaryFile(
+						file.filepath,
+						file.originalFilename ?? file.newFilename,
+						file.mimetype,
+					);
+
+					const binaryKey = `${binaryPropertyName}${fileIndex}`;
+
+					const binaryInfo = {
+						...pick(binaryFile, ['fileName', 'fileSize', 'fileType', 'mimeType', 'fileExtension']),
+						binaryKey,
+					};
+
+					returnItem.binary = Object.assign(returnItem.binary ?? {}, {
+						[`${binaryKey}`]: binaryFile,
+					});
+					returnItem.json.files = [
+						...(returnItem.json.files as Array<Omit<IBinaryData, 'data'>>),
+						binaryInfo,
+					];
+					fileIndex += 1;
+				}
+			}
+		}
+
+		return returnItem;
+	}
+
+	async webhook(ctx: IWebhookFunctions): Promise<IWebhookResponseData> {
+		const res = ctx.getResponseObject();
+
+		const isPublic = ctx.getNodeParameter('public', false) as boolean;
+		const nodeMode = ctx.getNodeParameter('mode', 'hostedChat') as string;
 		if (!isPublic) {
 			res.status(404).end();
 			return {
@@ -332,22 +444,25 @@ export class ChatTrigger implements INodeType {
 			};
 		}
 
-		const webhookName = this.getWebhookName();
-		const mode = this.getMode() === 'manual' ? 'test' : 'production';
-		const bodyData = this.getBodyData() ?? {};
-
-		const options = this.getNodeParameter('options', {}) as {
+		const options = ctx.getNodeParameter('options', {}) as {
 			getStarted?: string;
 			inputPlaceholder?: string;
 			loadPreviousSession?: LoadPreviousSessionChatOption;
 			showWelcomeScreen?: boolean;
 			subtitle?: string;
 			title?: string;
+			allowFileUploads?: boolean;
+			allowedFilesMimeTypes?: string;
 		};
 
+		const req = ctx.getRequestObject();
+		const webhookName = ctx.getWebhookName();
+		const mode = ctx.getMode() === 'manual' ? 'test' : 'production';
+		const bodyData = ctx.getBodyData() ?? {};
+
 		if (nodeMode === 'hostedChat') {
 			try {
-				await validateAuth(this);
+				await validateAuth(ctx);
 			} catch (error) {
 				if (error) {
 					res.writeHead((error as IDataObject).responseCode as number, {
@@ -361,19 +476,19 @@ export class ChatTrigger implements INodeType {
 
 			// Show the chat on GET request
 			if (webhookName === 'setup') {
-				const webhookUrlRaw = this.getNodeWebhookUrl('default') as string;
+				const webhookUrlRaw = ctx.getNodeWebhookUrl('default') as string;
 				const webhookUrl =
 					mode === 'test' ? webhookUrlRaw.replace('/webhook', '/webhook-test') : webhookUrlRaw;
-				const authentication = this.getNodeParameter('authentication') as
+				const authentication = ctx.getNodeParameter('authentication') as
 					| 'none'
 					| 'basicAuth'
 					| 'n8nUserAuth';
-				const initialMessagesRaw = this.getNodeParameter('initialMessages', '') as string;
+				const initialMessagesRaw = ctx.getNodeParameter('initialMessages', '') as string;
 				const initialMessages = initialMessagesRaw
 					.split('\n')
 					.filter((line) => line)
 					.map((line) => line.trim());
-				const instanceId = this.getInstanceId();
+				const instanceId = ctx.getInstanceId();
 
 				const i18nConfig = pick(options, ['getStarted', 'inputPlaceholder', 'subtitle', 'title']);
 
@@ -388,6 +503,8 @@ export class ChatTrigger implements INodeType {
 					mode,
 					instanceId,
 					authentication,
+					allowFileUploads: options.allowFileUploads,
+					allowedFilesMimeTypes: options.allowedFilesMimeTypes,
 				});
 
 				res.status(200).send(page).end();
@@ -399,7 +516,7 @@ export class ChatTrigger implements INodeType {
 
 		if (bodyData.action === 'loadPreviousSession') {
 			if (options?.loadPreviousSession === 'memory') {
-				const memory = (await this.getInputConnectionData(NodeConnectionType.AiMemory, 0)) as
+				const memory = (await ctx.getInputConnectionData(NodeConnectionType.AiMemory, 0)) as
 					| BaseChatMemory
 					| undefined;
 				const messages = ((await memory?.chatHistory.getMessages()) ?? [])
@@ -416,11 +533,21 @@ export class ChatTrigger implements INodeType {
 			}
 		}
 
-		const returnData: IDataObject = { ...bodyData };
+		let returnData: INodeExecutionData[];
 		const webhookResponse: IDataObject = { status: 200 };
+		if (req.contentType === 'multipart/form-data') {
+			returnData = [await this.handleFormData(ctx)];
+			return {
+				webhookResponse,
+				workflowData: [returnData],
+			};
+		} else {
+			returnData = [{ json: bodyData }];
+		}
+
 		return {
 			webhookResponse,
-			workflowData: [this.helpers.returnJsonArray(returnData)],
+			workflowData: [ctx.helpers.returnJsonArray(returnData)],
 		};
 	}
 }

packages/@n8n/nodes-langchain/nodes/trigger/ChatTrigger/templates.ts

@@ -8,6 +8,8 @@ export function createPage({
 	i18n: { en },
 	initialMessages,
 	authentication,
+	allowFileUploads,
+	allowedFilesMimeTypes,
 }: {
 	instanceId: string;
 	webhookUrl?: string;
@@ -19,6 +21,8 @@ export function createPage({
 	initialMessages: string[];
 	mode: 'test' | 'production';
 	authentication: AuthenticationChatOption;
+	allowFileUploads?: boolean;
+	allowedFilesMimeTypes?: string;
 }) {
 	const validAuthenticationOptions: AuthenticationChatOption[] = [
 		'none',
@@ -35,6 +39,8 @@ export function createPage({
 		? authentication
 		: 'none';
 	const sanitizedShowWelcomeScreen = !!showWelcomeScreen;
+	const sanitizedAllowFileUploads = !!allowFileUploads;
+	const sanitizedAllowedFilesMimeTypes = allowedFilesMimeTypes?.toString() ?? '';
 	const sanitizedLoadPreviousSession = validLoadPreviousSessionOptions.includes(
 		loadPreviousSession as LoadPreviousSessionChatOption,
 	)
@@ -103,6 +109,8 @@ export function createPage({
 								'X-Instance-Id': '${instanceId}',
 							}
 						},
+						allowFileUploads: ${sanitizedAllowFileUploads},
+						allowedFilesMimeTypes: '${sanitizedAllowedFilesMimeTypes}',
 						i18n: {
 							${en ? `en: ${JSON.stringify(en)},` : ''}
 						},