feat(core): Introduce object store service (#7225)

Depends on https://github.com/n8n-io/n8n/pull/7220 | Story:
[PAY-840](https://linear.app/n8n/issue/PAY-840/introduce-object-store-service-and-manager-for-binary-data)

This PR introduces an object store service for Enterprise edition. Note
that the service is tested but currently unused - it will be integrated
soon as a binary data manager, and later for execution data.
`amazonaws.com` in the host is temporarily hardcoded until we integrate
the service and test against AWS, Cloudflare and Backblaze, in the next
PR.

This is ready for review - the PR it depends on is approved and waiting
for CI.

---------

Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ <aditya@netroy.in>

packages/core/src/BinaryData/BinaryData.service.ts

@@ -1,13 +1,13 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+
 import { readFile, stat } from 'fs/promises';
-import concatStream from 'concat-stream';
 import prettyBytes from 'pretty-bytes';
 import { Service } from 'typedi';
 import { BINARY_ENCODING, LoggerProxy as Logger, IBinaryData } from 'n8n-workflow';
 
-import { FileSystemManager } from './FileSystem.manager';
 import { UnknownBinaryDataManager, InvalidBinaryDataMode } from './errors';
 import { LogCatch } from '../decorators/LogCatch.decorator';
-import { areValidModes } from './utils';
+import { areValidModes, toBuffer } from './utils';
 
 import type { Readable } from 'stream';
 import type { BinaryData } from './types';
@@ -15,8 +15,6 @@ import type { INodeExecutionData } from 'n8n-workflow';
 
 @Service()
 export class BinaryDataService {
-	private availableModes: BinaryData.Mode[] = [];
-
 	private mode: BinaryData.Mode = 'default';
 
 	private managers: Record<string, BinaryData.Manager> = {};
@@ -24,10 +22,10 @@ export class BinaryDataService {
 	async init(config: BinaryData.Config) {
 		if (!areValidModes(config.availableModes)) throw new InvalidBinaryDataMode();
 
-		this.availableModes = config.availableModes;
 		this.mode = config.mode;
 
-		if (this.availableModes.includes('filesystem')) {
+		if (config.availableModes.includes('filesystem')) {
+			const { FileSystemManager } = await import('./FileSystem.manager');
 			this.managers.filesystem = new FileSystemManager(config.localStoragePath);
 
 			await this.managers.filesystem.init();
@@ -80,7 +78,7 @@ export class BinaryDataService {
 		const manager = this.managers[this.mode];
 
 		if (!manager) {
-			const buffer = await this.binaryToBuffer(bufferOrStream);
+			const buffer = await this.toBuffer(bufferOrStream);
 			binaryData.data = buffer.toString(BINARY_ENCODING);
 			binaryData.fileSize = prettyBytes(buffer.length);
 
@@ -106,11 +104,8 @@ export class BinaryDataService {
 		return binaryData;
 	}
 
-	async binaryToBuffer(body: Buffer | Readable) {
-		return new Promise<Buffer>((resolve) => {
-			if (Buffer.isBuffer(body)) resolve(body);
-			else body.pipe(concatStream(resolve));
-		});
+	async toBuffer(bufferOrStream: Buffer | Readable) {
+		return toBuffer(bufferOrStream);
 	}
 
 	async getAsStream(binaryDataId: string, chunkSize?: number) {
@@ -141,12 +136,12 @@ export class BinaryDataService {
 		return this.getManager(mode).getMetadata(fileId);
 	}
 
-	async deleteManyByExecutionIds(executionIds: string[]) {
+	async deleteMany(ids: BinaryData.IdsForDeletion) {
 		const manager = this.managers[this.mode];
 
 		if (!manager) return;
 
-		await manager.deleteManyByExecutionIds(executionIds);
+		await manager.deleteMany(ids);
 	}
 
 	@LogCatch((error) =>

packages/core/src/BinaryData/FileSystem.manager.ts

@@ -59,7 +59,7 @@ export class FileSystemManager implements BinaryData.Manager {
 		bufferOrStream: Buffer | Readable,
 		{ mimeType, fileName }: BinaryData.PreWriteMetadata,
 	) {
-		const fileId = this.createFileId(executionId);
+		const fileId = this.toFileId(executionId);
 		const filePath = this.getPath(fileId);
 
 		await fs.writeFile(filePath, bufferOrStream);
@@ -77,10 +77,11 @@ export class FileSystemManager implements BinaryData.Manager {
 		return fs.rm(filePath);
 	}
 
-	async deleteManyByExecutionIds(executionIds: string[]) {
+	async deleteMany(ids: BinaryData.IdsForDeletion) {
+		const executionIds = ids.map((o) => o.executionId);
+
 		const set = new Set(executionIds);
 		const fileNames = await fs.readdir(this.storagePath);
-		const deletedIds = [];
 
 		for (const fileName of fileNames) {
 			const executionId = fileName.match(EXECUTION_ID_EXTRACTOR)?.[1];
@@ -89,12 +90,8 @@ export class FileSystemManager implements BinaryData.Manager {
 				const filePath = this.resolvePath(fileName);
 
 				await Promise.all([fs.rm(filePath), fs.rm(`${filePath}.metadata`)]);
-
-				deletedIds.push(executionId);
 			}
 		}
-
-		return deletedIds;
 	}
 
 	async copyByFilePath(
@@ -103,7 +100,7 @@ export class FileSystemManager implements BinaryData.Manager {
 		filePath: string,
 		{ mimeType, fileName }: BinaryData.PreWriteMetadata,
 	) {
-		const newFileId = this.createFileId(executionId);
+		const newFileId = this.toFileId(executionId);
 
 		await fs.cp(filePath, this.getPath(newFileId));
 
@@ -114,12 +111,14 @@ export class FileSystemManager implements BinaryData.Manager {
 		return { fileId: newFileId, fileSize };
 	}
 
-	async copyByFileId(_workflowId: string, executionId: string, fileId: string) {
-		const newFileId = this.createFileId(executionId);
+	async copyByFileId(_workflowId: string, executionId: string, sourceFileId: string) {
+		const targetFileId = this.toFileId(executionId);
+		const sourcePath = this.resolvePath(sourceFileId);
+		const targetPath = this.resolvePath(targetFileId);
 
-		await fs.copyFile(this.resolvePath(fileId), this.resolvePath(newFileId));
+		await fs.copyFile(sourcePath, targetPath);
 
-		return newFileId;
+		return targetFileId;
 	}
 
 	async rename(oldFileId: string, newFileId: string) {
@@ -136,7 +135,7 @@ export class FileSystemManager implements BinaryData.Manager {
 	//         private methods
 	// ----------------------------------
 
-	private createFileId(executionId: string) {
+	private toFileId(executionId: string) {
 		return [executionId, uuid()].join('');
 	}
 

packages/core/src/BinaryData/types.ts

@@ -20,6 +20,8 @@ export namespace BinaryData {
 
 	export type PreWriteMetadata = Omit<Metadata, 'fileSize'>;
 
+	export type IdsForDeletion = Array<{ workflowId: string; executionId: string }>;
+
 	export interface Manager {
 		init(): Promise<void>;
 
@@ -35,7 +37,7 @@ export namespace BinaryData {
 		getAsStream(fileId: string, chunkSize?: number): Promise<Readable>;
 		getMetadata(fileId: string): Promise<Metadata>;
 
-		copyByFileId(workflowId: string, executionId: string, fileId: string): Promise<string>;
+		copyByFileId(workflowId: string, executionId: string, sourceFileId: string): Promise<string>;
 		copyByFilePath(
 			workflowId: string,
 			executionId: string,
@@ -44,7 +46,7 @@ export namespace BinaryData {
 		): Promise<WriteResult>;
 
 		deleteOne(fileId: string): Promise<void>;
-		deleteManyByExecutionIds(executionIds: string[]): Promise<string[]>;
+		deleteMany(ids: IdsForDeletion): Promise<void>;
 
 		rename(oldFileId: string, newFileId: string): Promise<void>;
 	}

packages/core/src/BinaryData/utils.ts

@@ -1,5 +1,7 @@
-import fs from 'fs/promises';
+import fs from 'node:fs/promises';
+import type { Readable } from 'node:stream';
 import type { BinaryData } from './types';
+import concatStream from 'concat-stream';
 
 /**
  * Modes for storing binary data:
@@ -20,3 +22,10 @@ export async function ensureDirExists(dir: string) {
 		await fs.mkdir(dir, { recursive: true });
 	}
 }
+
+export async function toBuffer(body: Buffer | Readable) {
+	return new Promise<Buffer>((resolve) => {
+		if (Buffer.isBuffer(body)) resolve(body);
+		else body.pipe(concatStream(resolve));
+	});
+}