refactor(Google Drive Node): Use node streams for uploading and downloading files (#5017)

* use streams to upload files to google drive

* use streams to download files from google drive

* use resumable uploads api for google drive

* avoid dangling promises, and reduce memory usage in error logging
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™
2023-01-04 12:29:56 +01:00
committed by GitHub
parent 8b19fdd5f0
commit 54126b2c87
6 changed files with 229 additions and 163 deletions

View File

@@ -1,10 +1,11 @@
import { createReadStream } from 'fs';
import fs from 'fs/promises';
import { jsonParse } from 'n8n-workflow';
import path from 'path';
import { v4 as uuid } from 'uuid';
import type { Readable } from 'stream';
import { BinaryMetadata, jsonParse } from 'n8n-workflow';
import { BinaryMetadata, IBinaryDataConfig, IBinaryDataManager } from '../Interfaces';
import { IBinaryDataConfig, IBinaryDataManager } from '../Interfaces';
const PREFIX_METAFILE = 'binarymeta';
const PREFIX_PERSISTED_METAFILE = 'persistedmeta';
@@ -74,6 +75,10 @@ export class BinaryDataFileSystem implements IBinaryDataManager {
return binaryDataId;
}
getBinaryStream(identifier: string, chunkSize?: number): Readable {
return createReadStream(this.getBinaryPath(identifier), { highWaterMark: chunkSize });
}
async retrieveBinaryDataByIdentifier(identifier: string): Promise<Buffer> {
return this.retrieveFromLocalStorage(identifier);
}