mohiit1502/Automata
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(core): Lazy-load nodes and credentials to reduce baseline memory usage (#4577)

Browse Source
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™
2022-11-23 16:20:28 +01:00
committed by GitHub
parent f63cd3b89e
commit b6c57e19fc
71 changed files with 1102 additions and 1279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
packages/cli/src/commands/execute.ts
View File

@@ -123,19 +123,19 @@ export class Execute extends Command {
// Wait till the n8n-packages have been read
await loadNodesAndCredentialsPromise;
NodeTypes(loadNodesAndCredentials);
const credentialTypes = CredentialTypes(loadNodesAndCredentials);
// Load the credentials overwrites if any exist
const credentialsOverwrites = CredentialsOverwrites();
await credentialsOverwrites.init();
await CredentialsOverwrites(credentialTypes).init();
// Load all external hooks
const externalHooks = ExternalHooks();
await externalHooks.init();
// Add the found types to an instance other parts of the application can use
const nodeTypes = NodeTypes();
await nodeTypes.init(loadNodesAndCredentials.nodeTypes);
const credentialTypes = CredentialTypes();
await credentialTypes.init(loadNodesAndCredentials.credentialTypes);
const nodeTypes = NodeTypes(loadNodesAndCredentials);
CredentialTypes(loadNodesAndCredentials);
const instanceId = await UserSettings.getInstanceId();
const { cli } = await GenericHelpers.getVersions();

14
packages/cli/src/commands/executeBatch.ts
View File

@@ -17,8 +17,7 @@ import { sep } from 'path';
import { diff } from 'json-diff';
// eslint-disable-next-line import/no-extraneous-dependencies
import { pick } from 'lodash';
import pick from 'lodash.pick';
import { getLogger } from '@/Logger';
import * as ActiveExecutions from '@/ActiveExecutions';
@@ -312,18 +311,19 @@ export class ExecuteBatch extends Command {
// Wait till the n8n-packages have been read
await loadNodesAndCredentialsPromise;
NodeTypes(loadNodesAndCredentials);
const credentialTypes = CredentialTypes(loadNodesAndCredentials);
// Load the credentials overwrites if any exist
await CredentialsOverwrites().init();
await CredentialsOverwrites(credentialTypes).init();
// Load all external hooks
const externalHooks = ExternalHooks();
await externalHooks.init();
// Add the found types to an instance other parts of the application can use
const nodeTypes = NodeTypes();
await nodeTypes.init(loadNodesAndCredentials.nodeTypes);
const credentialTypes = CredentialTypes();
await credentialTypes.init(loadNodesAndCredentials.credentialTypes);
const nodeTypes = NodeTypes(loadNodesAndCredentials);
CredentialTypes(loadNodesAndCredentials);
const instanceId = await UserSettings.getInstanceId();
const { cli } = await GenericHelpers.getVersions();

24
packages/cli/src/commands/start.ts
View File

@@ -9,7 +9,6 @@
import localtunnel from 'localtunnel';
import { BinaryDataManager, TUNNEL_SUBDOMAIN_ENV, UserSettings } from 'n8n-core';
import { Command, flags } from '@oclif/command';
// eslint-disable-next-line import/no-extraneous-dependencies
import Redis from 'ioredis';
import { IDataObject, LoggerProxy, sleep } from 'n8n-workflow';
@@ -115,7 +114,7 @@ export class Start extends Command {
await InternalHooksManager.getInstance().onN8nStop();
const skipWebhookDeregistration = config.getEnv(
'endpoints.skipWebhoooksDeregistrationOnShutdown',
'endpoints.skipWebhooksDeregistrationOnShutdown',
);
const removePromises = [];
@@ -210,14 +209,13 @@ export class Start extends Command {
await externalHooks.init();
// Add the found types to an instance other parts of the application can use
const nodeTypes = NodeTypes();
await nodeTypes.init(loadNodesAndCredentials.nodeTypes);
const credentialTypes = CredentialTypes();
await credentialTypes.init(loadNodesAndCredentials.credentialTypes);
const nodeTypes = NodeTypes(loadNodesAndCredentials);
const credentialTypes = CredentialTypes(loadNodesAndCredentials);
// Load the credentials overwrites if any exist
const credentialsOverwrites = CredentialsOverwrites();
await credentialsOverwrites.init();
await CredentialsOverwrites(credentialTypes).init();
await loadNodesAndCredentials.generateTypesForFrontend();
// Wait till the database is ready
await startDbInitPromise;
@@ -227,13 +225,13 @@ export class Start extends Command {
packageName: string;
version: string;
}>();
installedPackages.forEach((installedpackage) => {
installedpackage.installedNodes.forEach((installedNode) => {
if (!loadNodesAndCredentials.nodeTypes[installedNode.type]) {
installedPackages.forEach((installedPackage) => {
installedPackage.installedNodes.forEach((installedNode) => {
if (!loadNodesAndCredentials.known.nodes[installedNode.type]) {
// Leave the list ready for installing in case we need.
missingPackages.add({
packageName: installedpackage.packageName,
version: installedpackage.installedVersion,
packageName: installedPackage.packageName,
version: installedPackage.installedVersion,
});
}
});

16
packages/cli/src/commands/webhook.ts
View File

@@ -6,7 +6,6 @@
/* eslint-disable @typescript-eslint/unbound-method */
import { BinaryDataManager, UserSettings } from 'n8n-core';
import { Command, flags } from '@oclif/command';
// eslint-disable-next-line import/no-extraneous-dependencies
import Redis from 'ioredis';
import { IDataObject, LoggerProxy, sleep } from 'n8n-workflow';
@@ -132,26 +131,23 @@ export class Webhook extends Command {
// Make sure the settings exist
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const userSettings = await UserSettings.prepareUserSettings();
await UserSettings.prepareUserSettings();
// Load all node and credential types
const loadNodesAndCredentials = LoadNodesAndCredentials();
await loadNodesAndCredentials.init();
// Add the found types to an instance other parts of the application can use
const nodeTypes = NodeTypes(loadNodesAndCredentials);
const credentialTypes = CredentialTypes(loadNodesAndCredentials);
// Load the credentials overwrites if any exist
const credentialsOverwrites = CredentialsOverwrites();
await credentialsOverwrites.init();
await CredentialsOverwrites(credentialTypes).init();
// Load all external hooks
const externalHooks = ExternalHooks();
await externalHooks.init();
// Add the found types to an instance other parts of the application can use
const nodeTypes = NodeTypes();
await nodeTypes.init(loadNodesAndCredentials.nodeTypes);
const credentialTypes = CredentialTypes();
await credentialTypes.init(loadNodesAndCredentials.credentialTypes);
// Wait till the database is ready
await startDbInitPromise;

14
packages/cli/src/commands/worker.ts
View File

@@ -7,7 +7,6 @@
/* eslint-disable @typescript-eslint/no-non-null-assertion */
/* eslint-disable @typescript-eslint/restrict-template-expressions */
/* eslint-disable @typescript-eslint/no-unused-vars */
// eslint-disable-next-line import/no-extraneous-dependencies
import express from 'express';
import http from 'http';
import PCancelable from 'p-cancelable';
@@ -283,20 +282,17 @@ export class Worker extends Command {
const loadNodesAndCredentials = LoadNodesAndCredentials();
await loadNodesAndCredentials.init();
// Add the found types to an instance other parts of the application can use
const nodeTypes = NodeTypes(loadNodesAndCredentials);
const credentialTypes = CredentialTypes(loadNodesAndCredentials);
// Load the credentials overwrites if any exist
const credentialsOverwrites = CredentialsOverwrites();
await credentialsOverwrites.init();
await CredentialsOverwrites(credentialTypes).init();
// Load all external hooks
const externalHooks = ExternalHooks();
await externalHooks.init();
// Add the found types to an instance other parts of the application can use
const nodeTypes = NodeTypes();
await nodeTypes.init(loadNodesAndCredentials.nodeTypes);
const credentialTypes = CredentialTypes();
await credentialTypes.init(loadNodesAndCredentials.credentialTypes);
// Wait till the database is ready
await startDbInitPromise;