mohiit1502/Automata
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(core): Introduce object store service (#7225)

Browse Source 
Depends on https://github.com/n8n-io/n8n/pull/7220 | Story:
[PAY-840](https://linear.app/n8n/issue/PAY-840/introduce-object-store-service-and-manager-for-binary-data)

This PR introduces an object store service for Enterprise edition. Note
that the service is tested but currently unused - it will be integrated
soon as a binary data manager, and later for execution data.
`amazonaws.com` in the host is temporarily hardcoded until we integrate
the service and test against AWS, Cloudflare and Backblaze, in the next
PR.

This is ready for review - the PR it depends on is approved and waiting
for CI.

---------

Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ <aditya@netroy.in>
This commit is contained in:
Iván Ovejero
2023-09-27 09:42:35 +02:00
committed by GitHub
parent 2c4e25c06b
commit fa845453bb
15 changed files with 12628 additions and 4086 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
packages/core/src/BinaryData/FileSystem.manager.ts
View File

@@ -59,7 +59,7 @@ export class FileSystemManager implements BinaryData.Manager {
bufferOrStream: Buffer | Readable,
{ mimeType, fileName }: BinaryData.PreWriteMetadata,
) {
const fileId = this.createFileId(executionId);
const fileId = this.toFileId(executionId);
const filePath = this.getPath(fileId);
await fs.writeFile(filePath, bufferOrStream);
@@ -77,10 +77,11 @@ export class FileSystemManager implements BinaryData.Manager {
return fs.rm(filePath);
}
async deleteManyByExecutionIds(executionIds: string[]) {
async deleteMany(ids: BinaryData.IdsForDeletion) {
const executionIds = ids.map((o) => o.executionId);
const set = new Set(executionIds);
const fileNames = await fs.readdir(this.storagePath);
const deletedIds = [];
for (const fileName of fileNames) {
const executionId = fileName.match(EXECUTION_ID_EXTRACTOR)?.[1];
@@ -89,12 +90,8 @@ export class FileSystemManager implements BinaryData.Manager {
const filePath = this.resolvePath(fileName);
await Promise.all([fs.rm(filePath), fs.rm(`${filePath}.metadata`)]);
deletedIds.push(executionId);
}
}
return deletedIds;
}
async copyByFilePath(
@@ -103,7 +100,7 @@ export class FileSystemManager implements BinaryData.Manager {
filePath: string,
{ mimeType, fileName }: BinaryData.PreWriteMetadata,
) {
const newFileId = this.createFileId(executionId);
const newFileId = this.toFileId(executionId);
await fs.cp(filePath, this.getPath(newFileId));
@@ -114,12 +111,14 @@ export class FileSystemManager implements BinaryData.Manager {
return { fileId: newFileId, fileSize };
}
async copyByFileId(_workflowId: string, executionId: string, fileId: string) {
const newFileId = this.createFileId(executionId);
async copyByFileId(_workflowId: string, executionId: string, sourceFileId: string) {
const targetFileId = this.toFileId(executionId);
const sourcePath = this.resolvePath(sourceFileId);
const targetPath = this.resolvePath(targetFileId);
await fs.copyFile(this.resolvePath(fileId), this.resolvePath(newFileId));
await fs.copyFile(sourcePath, targetPath);
return newFileId;
return targetFileId;
}
async rename(oldFileId: string, newFileId: string) {
@@ -136,7 +135,7 @@ export class FileSystemManager implements BinaryData.Manager {
// private methods
// ----------------------------------
private createFileId(executionId: string) {
private toFileId(executionId: string) {
return [executionId, uuid()].join('');
}