refactor(core): Reduce memory usage in the Webhook node (#4640)

use file streaming to pass webhook binaries around

packages/core/src/BinaryDataManager/FileSystem.ts

@@ -1,8 +1,9 @@
-import { promises as fs } from 'fs';
+import fs from 'fs/promises';
+import { jsonParse } from 'n8n-workflow';
 import path from 'path';
 import { v4 as uuid } from 'uuid';
 
-import { IBinaryDataConfig, IBinaryDataManager } from '../Interfaces';
+import { BinaryMetadata, IBinaryDataConfig, IBinaryDataManager } from '../Interfaces';
 
 const PREFIX_METAFILE = 'binarymeta';
 const PREFIX_PERSISTED_METAFILE = 'persistedmeta';
@@ -43,17 +44,47 @@ export class BinaryDataFileSystem implements IBinaryDataManager {
 			.then(() => {});
 	}
 
+	async getFileSize(identifier: string): Promise<number> {
+		const stats = await fs.stat(this.getBinaryPath(identifier));
+		return stats.size;
+	}
+
+	async copyBinaryFile(filePath: string, executionId: string): Promise<string> {
+		const binaryDataId = this.generateFileName(executionId);
+		await this.addBinaryIdToPersistMeta(executionId, binaryDataId);
+		await this.copyFileToLocalStorage(filePath, binaryDataId);
+		return binaryDataId;
+	}
+
+	async storeBinaryMetadata(identifier: string, metadata: BinaryMetadata) {
+		await fs.writeFile(this.getMetadataPath(identifier), JSON.stringify(metadata), {
+			encoding: 'utf-8',
+		});
+	}
+
+	async getBinaryMetadata(identifier: string): Promise<BinaryMetadata> {
+		return jsonParse(await fs.readFile(this.getMetadataPath(identifier), { encoding: 'utf-8' }));
+	}
+
 	async storeBinaryData(binaryBuffer: Buffer, executionId: string): Promise<string> {
 		const binaryDataId = this.generateFileName(executionId);
-		return this.addBinaryIdToPersistMeta(executionId, binaryDataId).then(async () =>
-			this.saveToLocalStorage(binaryBuffer, binaryDataId).then(() => binaryDataId),
-		);
+		await this.addBinaryIdToPersistMeta(executionId, binaryDataId);
+		await this.saveToLocalStorage(binaryBuffer, binaryDataId);
+		return binaryDataId;
 	}
 
 	async retrieveBinaryDataByIdentifier(identifier: string): Promise<Buffer> {
 		return this.retrieveFromLocalStorage(identifier);
 	}
 
+	getBinaryPath(identifier: string): string {
+		return path.join(this.storagePath, identifier);
+	}
+
+	getMetadataPath(identifier: string): string {
+		return path.join(this.storagePath, `${identifier}.metadata`);
+	}
+
 	async markDataForDeletionByExecutionId(executionId: string): Promise<void> {
 		const tt = new Date(new Date().getTime() + this.binaryDataTTL * 60000);
 		return fs.writeFile(
@@ -180,7 +211,7 @@ export class BinaryDataFileSystem implements IBinaryDataManager {
 	}
 
 	private generateFileName(prefix: string): string {
-		return `${prefix}_${uuid()}`;
+		return [prefix, uuid()].join('');
 	}
 
 	private getBinaryDataMetaPath() {
@@ -196,15 +227,19 @@ export class BinaryDataFileSystem implements IBinaryDataManager {
 	}
 
 	private async deleteFromLocalStorage(identifier: string) {
-		return fs.rm(path.join(this.storagePath, identifier));
+		return fs.rm(this.getBinaryPath(identifier));
+	}
+
+	private async copyFileToLocalStorage(source: string, identifier: string): Promise<void> {
+		await fs.cp(source, this.getBinaryPath(identifier));
 	}
 
 	private async saveToLocalStorage(data: Buffer, identifier: string) {
-		await fs.writeFile(path.join(this.storagePath, identifier), data);
+		await fs.writeFile(this.getBinaryPath(identifier), data);
 	}
 
 	private async retrieveFromLocalStorage(identifier: string): Promise<Buffer> {
-		const filePath = path.join(this.storagePath, identifier);
+		const filePath = this.getBinaryPath(identifier);
 		try {
 			return await fs.readFile(filePath);
 		} catch (e) {