feat(core): Lazy-load nodes and credentials to reduce baseline memory usage (#4577)

packages/cli/src/ActiveExecutions.ts

@@ -14,7 +14,6 @@ import {
 
 import { ChildProcess } from 'child_process';
 import { stringify } from 'flatted';
-// eslint-disable-next-line import/no-extraneous-dependencies
 import PCancelable from 'p-cancelable';
 import * as Db from '@/Db';
 import {

packages/cli/src/ActiveWorkflowRunner.ts

@@ -86,7 +86,7 @@ export class ActiveWorkflowRunner {
 			relations: ['shared', 'shared.user', 'shared.user.globalRole'],
 		})) as IWorkflowDb[];
 
-		if (!config.getEnv('endpoints.skipWebhoooksDeregistrationOnShutdown')) {
+		if (!config.getEnv('endpoints.skipWebhooksDeregistrationOnShutdown')) {
 			// Do not clean up database when skip registration is done.
 			// This flag is set when n8n is running in scaled mode.
 			// Impact is minimal, but for a short while, n8n will stop accepting requests.
@@ -401,7 +401,6 @@ export class ActiveWorkflowRunner {
 
 	/**
 	 * Adds all the webhooks of the workflow
-	 *
 	 */
 	async addWorkflowWebhooks(
 		workflow: Workflow,
@@ -462,7 +461,7 @@ export class ActiveWorkflowRunner {
 			} catch (error) {
 				if (
 					activation === 'init' &&
-					config.getEnv('endpoints.skipWebhoooksDeregistrationOnShutdown') &&
+					config.getEnv('endpoints.skipWebhooksDeregistrationOnShutdown') &&
 					error.name === 'QueryFailedError'
 				) {
 					// When skipWebhooksDeregistrationOnShutdown is enabled,
@@ -487,7 +486,10 @@ export class ActiveWorkflowRunner {
 				// TODO check if there is standard error code for duplicate key violation that works
 				// with all databases
 				if (error.name === 'QueryFailedError') {
-					error.message = `The URL path that the "${webhook.node}" node uses is already taken. Please change it to something else.`;
+					error = new Error(
+						`The URL path that the "${webhook.node}" node uses is already taken. Please change it to something else.`,
+						{ cause: error },
+					);
 				} else if (error.detail) {
 					// it's a error running the webhook methods (checkExists, create)
 					error.message = error.detail;

packages/cli/src/CommunityNodes/helpers.ts

@@ -3,7 +3,6 @@
 import { promisify } from 'util';
 import { exec } from 'child_process';
 import { access as fsAccess, mkdir as fsMkdir } from 'fs/promises';
-import { createContext, Script } from 'vm';
 import axios from 'axios';
 import { UserSettings } from 'n8n-core';
 import { LoggerProxy, PublicInstalledPackage } from 'n8n-workflow';
@@ -234,13 +233,3 @@ export const isClientError = (error: Error): boolean => {
 export function isNpmError(error: unknown): error is { code: number; stdout: string } {
 	return typeof error === 'object' && error !== null && 'code' in error && 'stdout' in error;
 }
-
-const context = createContext({ require });
-export const loadClassInIsolation = (filePath: string, className: string) => {
-	if (process.platform === 'win32') {
-		filePath = filePath.replace(/\\/g, '/');
-	}
-	const script = new Script(`new (require('${filePath}').${className})()`);
-	// eslint-disable-next-line @typescript-eslint/no-unsafe-return
-	return script.runInContext(context);
-};

packages/cli/src/CredentialTypes.ts

@@ -1,36 +1,58 @@
-import {
+import { loadClassInIsolation } from 'n8n-core';
+import type {
 	ICredentialType,
-	ICredentialTypeData,
-	ICredentialTypes as ICredentialTypesInterface,
+	ICredentialTypes,
+	INodesAndCredentials,
+	LoadedClass,
 } from 'n8n-workflow';
-import { RESPONSE_ERROR_MESSAGES } from '@/constants';
+import { RESPONSE_ERROR_MESSAGES } from './constants';
 
-class CredentialTypesClass implements ICredentialTypesInterface {
-	credentialTypes: ICredentialTypeData = {};
+class CredentialTypesClass implements ICredentialTypes {
+	constructor(private nodesAndCredentials: INodesAndCredentials) {}
 
-	async init(credentialTypes: ICredentialTypeData): Promise<void> {
-		this.credentialTypes = credentialTypes;
-	}
-
-	getAll(): ICredentialType[] {
-		return Object.values(this.credentialTypes).map((data) => data.type);
+	recognizes(type: string) {
+		return type in this.knownCredentials || type in this.loadedCredentials;
 	}
 
 	getByName(credentialType: string): ICredentialType {
-		try {
-			return this.credentialTypes[credentialType].type;
-		} catch (error) {
-			throw new Error(`${RESPONSE_ERROR_MESSAGES.NO_CREDENTIAL}: ${credentialType}`);
+		return this.getCredential(credentialType).type;
+	}
+
+	private getCredential(type: string): LoadedClass<ICredentialType> {
+		const loadedCredentials = this.loadedCredentials;
+		if (type in loadedCredentials) {
+			return loadedCredentials[type];
 		}
+
+		const knownCredentials = this.knownCredentials;
+		if (type in knownCredentials) {
+			const { className, sourcePath } = knownCredentials[type];
+			const loaded: ICredentialType = loadClassInIsolation(sourcePath, className);
+			loadedCredentials[type] = { sourcePath, type: loaded };
+			return loadedCredentials[type];
+		}
+		throw new Error(`${RESPONSE_ERROR_MESSAGES.NO_CREDENTIAL}: ${type}`);
+	}
+
+	private get loadedCredentials() {
+		return this.nodesAndCredentials.loaded.credentials;
+	}
+
+	private get knownCredentials() {
+		return this.nodesAndCredentials.known.credentials;
 	}
 }
 
 let credentialTypesInstance: CredentialTypesClass | undefined;
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
-export function CredentialTypes(): CredentialTypesClass {
-	if (credentialTypesInstance === undefined) {
-		credentialTypesInstance = new CredentialTypesClass();
+export function CredentialTypes(nodesAndCredentials?: INodesAndCredentials): CredentialTypesClass {
+	if (!credentialTypesInstance) {
+		if (nodesAndCredentials) {
+			credentialTypesInstance = new CredentialTypesClass(nodesAndCredentials);
+		} else {
+			throw new Error('CredentialTypes not initialized yet');
+		}
 	}
 
 	return credentialTypesInstance;

packages/cli/src/CredentialsHelper.ts

@@ -7,8 +7,7 @@
 /* eslint-disable @typescript-eslint/no-unsafe-call */
 
 import { Credentials, NodeExecuteFunctions } from 'n8n-core';
-// eslint-disable-next-line import/no-extraneous-dependencies
-import { get } from 'lodash';
+import get from 'lodash.get';
 
 import {
 	ICredentialDataDecryptedObject,
@@ -25,8 +24,6 @@ import {
 	INodeParameters,
 	INodeProperties,
 	INodeType,
-	INodeTypeData,
-	INodeTypes,
 	IVersionedNodeType,
 	VersionedNodeType,
 	IRequestOptionsSimplified,
@@ -40,6 +37,8 @@ import {
 	LoggerProxy as Logger,
 	ErrorReporterProxy as ErrorReporter,
 	IHttpRequestHelper,
+	INodeTypeData,
+	INodeTypes,
 } from 'n8n-workflow';
 
 import * as Db from '@/Db';
@@ -52,19 +51,16 @@ import { CredentialsOverwrites } from '@/CredentialsOverwrites';
 import { CredentialTypes } from '@/CredentialTypes';
 import { whereClause } from './UserManagement/UserManagementHelper';
 
+const mockNodesData: INodeTypeData = {};
 const mockNodeTypes: INodeTypes = {
-	nodeTypes: {} as INodeTypeData,
-	// eslint-disable-next-line @typescript-eslint/no-unused-vars
-	init: async (nodeTypes?: INodeTypeData): Promise<void> => {},
 	getAll(): Array<INodeType | IVersionedNodeType> {
-		// @ts-ignore
-		return Object.values(this.nodeTypes).map((data) => data.type);
+		return Object.values(mockNodesData).map((data) => data.type);
 	},
 	getByNameAndVersion(nodeType: string, version?: number): INodeType | undefined {
-		if (this.nodeTypes[nodeType] === undefined) {
+		if (mockNodesData[nodeType] === undefined) {
 			return undefined;
 		}
-		return NodeHelpers.getVersionedNodeType(this.nodeTypes[nodeType].type, version);
+		return NodeHelpers.getVersionedNodeType(mockNodesData[nodeType].type, version);
 	},
 };
 
@@ -623,21 +619,16 @@ export class CredentialsHelper extends ICredentialsHelper {
 			},
 		};
 
-		const nodeTypes: INodeTypes = {
-			...mockNodeTypes,
-			nodeTypes: {
-				[nodeTypeCopy.description.name]: {
-					sourcePath: '',
-					type: nodeTypeCopy,
-				},
-			},
+		mockNodesData[nodeTypeCopy.description.name] = {
+			sourcePath: '',
+			type: nodeTypeCopy,
 		};
 
 		const workflow = new Workflow({
 			nodes: workflowData.nodes,
 			connections: workflowData.connections,
 			active: false,
-			nodeTypes,
+			nodeTypes: mockNodeTypes,
 		});
 
 		const mode = 'internal';
@@ -719,6 +710,8 @@ export class CredentialsHelper extends ICredentialsHelper {
 				status: 'Error',
 				message: error.message.toString(),
 			};
+		} finally {
+			delete mockNodesData[nodeTypeCopy.description.name];
 		}
 
 		if (

packages/cli/src/CredentialsOverwrites.ts

@@ -1,51 +1,36 @@
-/* eslint-disable @typescript-eslint/no-unsafe-argument */
-/* eslint-disable no-underscore-dangle */
-import { deepCopy, ICredentialDataDecryptedObject } from 'n8n-workflow';
-import { CredentialTypes } from '@/CredentialTypes';
+import type { ICredentialDataDecryptedObject, ICredentialTypes } from 'n8n-workflow';
+import { deepCopy, LoggerProxy as Logger, jsonParse } from 'n8n-workflow';
 import type { ICredentialsOverwrite } from '@/Interfaces';
 import * as GenericHelpers from '@/GenericHelpers';
 
 class CredentialsOverwritesClass {
-	private credentialTypes = CredentialTypes();
-
 	private overwriteData: ICredentialsOverwrite = {};
 
 	private resolvedTypes: string[] = [];
 
-	async init(overwriteData?: ICredentialsOverwrite) {
+	constructor(private credentialTypes: ICredentialTypes) {}
+
+	async init() {
+		const data = (await GenericHelpers.getConfigValue('credentials.overwrite.data')) as string;
+
+		const overwriteData = jsonParse<ICredentialsOverwrite>(data, {
+			errorMessage: 'The credentials-overwrite is not valid JSON.',
+		});
+
+		this.setData(overwriteData);
+	}
+
+	setData(overwriteData: ICredentialsOverwrite) {
 		// If data gets reinitialized reset the resolved types cache
 		this.resolvedTypes.length = 0;
 
-		if (overwriteData !== undefined) {
-			// If data is already given it can directly be set instead of
-			// loaded from environment
-			this.__setData(deepCopy(overwriteData));
-			return;
-		}
-
-		const data = (await GenericHelpers.getConfigValue('credentials.overwrite.data')) as string;
-
-		try {
-			// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-shadow
-			const overwriteData = JSON.parse(data);
-			this.__setData(overwriteData);
-		} catch (error) {
-			throw new Error(`The credentials-overwrite is not valid JSON.`);
-		}
-	}
-
-	__setData(overwriteData: ICredentialsOverwrite) {
 		this.overwriteData = overwriteData;
 
-		// eslint-disable-next-line no-restricted-syntax
-		for (const credentialTypeData of this.credentialTypes.getAll()) {
-			const type = credentialTypeData.name;
-
-			const overwrites = this.__getExtended(type);
+		for (const type in overwriteData) {
+			const overwrites = this.getOverwrites(type);
 
 			if (overwrites && Object.keys(overwrites).length) {
 				this.overwriteData[type] = overwrites;
-				credentialTypeData.__overwrittenProperties = Object.keys(overwrites);
 			}
 		}
 	}
@@ -70,18 +55,19 @@ class CredentialsOverwritesClass {
 		return returnData;
 	}
 
-	__getExtended(type: string): ICredentialDataDecryptedObject | undefined {
+	private getOverwrites(type: string): ICredentialDataDecryptedObject | undefined {
 		if (this.resolvedTypes.includes(type)) {
 			// Type got already resolved and can so returned directly
 			return this.overwriteData[type];
 		}
 
-		const credentialTypeData = this.credentialTypes.getByName(type);
-
-		if (credentialTypeData === undefined) {
-			throw new Error(`The credentials of type "${type}" are not known.`);
+		if (!this.credentialTypes.recognizes(type)) {
+			Logger.warn(`Unknown credential type ${type} in Credential overwrites`);
+			return;
 		}
 
+		const credentialTypeData = this.credentialTypes.getByName(type);
+
 		if (credentialTypeData.extends === undefined) {
 			this.resolvedTypes.push(type);
 			return this.overwriteData[type];
@@ -90,7 +76,7 @@ class CredentialsOverwritesClass {
 		const overwrites: ICredentialDataDecryptedObject = {};
 		// eslint-disable-next-line no-restricted-syntax
 		for (const credentialsTypeName of credentialTypeData.extends) {
-			Object.assign(overwrites, this.__getExtended(credentialsTypeName));
+			Object.assign(overwrites, this.getOverwrites(credentialsTypeName));
 		}
 
 		if (this.overwriteData[type] !== undefined) {
@@ -102,7 +88,7 @@ class CredentialsOverwritesClass {
 		return overwrites;
 	}
 
-	get(type: string): ICredentialDataDecryptedObject | undefined {
+	private get(type: string): ICredentialDataDecryptedObject | undefined {
 		return this.overwriteData[type];
 	}
 
@@ -114,9 +100,15 @@ class CredentialsOverwritesClass {
 let credentialsOverwritesInstance: CredentialsOverwritesClass | undefined;
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
-export function CredentialsOverwrites(): CredentialsOverwritesClass {
-	if (credentialsOverwritesInstance === undefined) {
-		credentialsOverwritesInstance = new CredentialsOverwritesClass();
+export function CredentialsOverwrites(
+	credentialTypes?: ICredentialTypes,
+): CredentialsOverwritesClass {
+	if (!credentialsOverwritesInstance) {
+		if (credentialTypes) {
+			credentialsOverwritesInstance = new CredentialsOverwritesClass(credentialTypes);
+		} else {
+			throw new Error('CredentialsOverwrites not initialized yet');
+		}
 	}
 
 	return credentialsOverwritesInstance;

packages/cli/src/GenericHelpers.ts

@@ -7,6 +7,7 @@
 import express from 'express';
 import { join as pathJoin } from 'path';
 import { readFile as fsReadFile } from 'fs/promises';
+import type { n8n } from 'n8n-core';
 import {
 	ExecutionError,
 	IDataObject,
@@ -25,7 +26,6 @@ import {
 	IExecutionFlattedDb,
 	IPackageVersions,
 	IWorkflowDb,
-	IN8nNodePackageJson,
 } from '@/Interfaces';
 import * as ResponseHelper from '@/ResponseHelper';
 // eslint-disable-next-line import/order
@@ -64,7 +64,6 @@ export function getSessionId(req: express.Request): string | undefined {
 
 /**
  * Returns information which version of the packages are installed
- *
  */
 export async function getVersions(): Promise<IPackageVersions> {
 	if (versionCache !== undefined) {
@@ -72,11 +71,9 @@ export async function getVersions(): Promise<IPackageVersions> {
 	}
 
 	const packageFile = await fsReadFile(pathJoin(CLI_DIR, 'package.json'), 'utf8');
-	// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
-	const packageData = jsonParse<IN8nNodePackageJson>(packageFile);
+	const packageData = jsonParse<n8n.PackageJson>(packageFile);
 
 	versionCache = {
-		// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
 		cli: packageData.version,
 	};
 

packages/cli/src/Interfaces.ts

@@ -1,5 +1,5 @@
 /* eslint-disable @typescript-eslint/naming-convention */
-import {
+import type {
 	ExecutionError,
 	ICredentialDataDecryptedObject,
 	ICredentialsDecrypted,
@@ -15,6 +15,7 @@ import {
 	ITelemetrySettings,
 	ITelemetryTrackProperties,
 	IWorkflowBase as IWorkflowBaseWorkflow,
+	LoadingDetails,
 	Workflow,
 	WorkflowActivateMode,
 	WorkflowExecuteMode,
@@ -22,7 +23,6 @@ import {
 
 import { WorkflowExecute } from 'n8n-core';
 
-// eslint-disable-next-line import/no-extraneous-dependencies
 import PCancelable from 'p-cancelable';
 import type { FindOperator, Repository } from 'typeorm';
 
@@ -59,10 +59,7 @@ export interface ICustomRequest extends Request {
 }
 
 export interface ICredentialsTypeData {
-	[key: string]: {
-		className: string;
-		sourcePath: string;
-	};
+	[key: string]: LoadingDetails;
 }
 
 export interface ICredentialsOverwrite {
@@ -451,19 +448,6 @@ export interface IVersionNotificationSettings {
 	infoUrl: string;
 }
 
-export interface IN8nNodePackageJson {
-	name: string;
-	version: string;
-	n8n?: {
-		credentials?: string[];
-		nodes?: string[];
-	};
-	author?: {
-		name?: string;
-		email?: string;
-	};
-}
-
 export interface IN8nUISettings {
 	endpointWebhook: string;
 	endpointWebhookTest: string;
@@ -649,7 +633,7 @@ export interface IResponseCallbackData {
 	responseCode?: number;
 }
 
-export interface ITransferNodeTypes {
+export interface INodesTypeData {
 	[key: string]: {
 		className: string;
 		sourcePath: string;
@@ -697,10 +681,7 @@ export interface IWorkflowExecutionDataProcess {
 }
 
 export interface IWorkflowExecutionDataProcessWithExecution extends IWorkflowExecutionDataProcess {
-	credentialsOverwrite: ICredentialsOverwrite;
-	credentialsTypeData: ICredentialsTypeData;
 	executionId: string;
-	nodeTypeData: ITransferNodeTypes;
 	userId: string;
 }
 

packages/cli/src/LoadNodesAndCredentials.ts

@@ -1,132 +1,108 @@
-/* eslint-disable no-underscore-dangle */
-/* eslint-disable @typescript-eslint/naming-convention */
-/* eslint-disable no-prototype-builtins */
-/* eslint-disable no-param-reassign */
-/* eslint-disable @typescript-eslint/prefer-optional-chain */
-/* eslint-disable @typescript-eslint/no-unsafe-member-access */
-/* eslint-disable @typescript-eslint/no-unsafe-call */
-/* eslint-disable @typescript-eslint/no-unsafe-assignment */
-/* eslint-disable no-await-in-loop */
-/* eslint-disable no-continue */
-/* eslint-disable no-restricted-syntax */
-import { CUSTOM_EXTENSION_ENV, UserSettings } from 'n8n-core';
 import {
-	CodexData,
-	ICredentialType,
-	ICredentialTypeData,
+	CUSTOM_EXTENSION_ENV,
+	UserSettings,
+	CustomDirectoryLoader,
+	DirectoryLoader,
+	PackageDirectoryLoader,
+	LazyPackageDirectoryLoader,
+	Types,
+} from 'n8n-core';
+import type {
 	ILogger,
-	INodeType,
-	INodeTypeData,
-	INodeTypeNameVersion,
-	IVersionedNodeType,
-	LoggerProxy,
-	jsonParse,
-	ErrorReporterProxy as ErrorReporter,
+	INodesAndCredentials,
+	KnownNodesAndCredentials,
+	LoadedNodesAndCredentials,
 } from 'n8n-workflow';
+import { LoggerProxy, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow';
 
 import {
 	access as fsAccess,
+	copyFile,
+	mkdir,
 	readdir as fsReaddir,
-	readFile as fsReadFile,
 	stat as fsStat,
+	writeFile,
 } from 'fs/promises';
-import glob from 'fast-glob';
 import path from 'path';
-import pick from 'lodash.pick';
-import { IN8nNodePackageJson } from '@/Interfaces';
-import { getLogger } from '@/Logger';
 import config from '@/config';
-import { NodeTypes } from '@/NodeTypes';
 import { InstalledPackages } from '@db/entities/InstalledPackages';
 import { InstalledNodes } from '@db/entities/InstalledNodes';
-import { executeCommand, loadClassInIsolation } from '@/CommunityNodes/helpers';
-import { CLI_DIR, RESPONSE_ERROR_MESSAGES } from '@/constants';
+import { executeCommand } from '@/CommunityNodes/helpers';
+import { CLI_DIR, GENERATED_STATIC_DIR, RESPONSE_ERROR_MESSAGES } from '@/constants';
 import {
 	persistInstalledPackageData,
 	removePackageFromDatabase,
 } from '@/CommunityNodes/packageModel';
+import { CredentialsOverwrites } from '@/CredentialsOverwrites';
 
-const CUSTOM_NODES_CATEGORY = 'Custom Nodes';
+export class LoadNodesAndCredentialsClass implements INodesAndCredentials {
+	known: KnownNodesAndCredentials = { nodes: {}, credentials: {} };
 
-function toJSON() {
-	// eslint-disable-next-line @typescript-eslint/no-unsafe-return
-	return {
-		...this,
-		authenticate: typeof this.authenticate === 'function' ? {} : this.authenticate,
-	};
-}
+	loaded: LoadedNodesAndCredentials = { nodes: {}, credentials: {} };
 
-class LoadNodesAndCredentialsClass {
-	nodeTypes: INodeTypeData = {};
+	types: Types = { nodes: [], credentials: [] };
 
-	credentialTypes: ICredentialTypeData = {};
+	excludeNodes = config.getEnv('nodes.exclude');
 
-	excludeNodes: string | undefined = undefined;
-
-	includeNodes: string | undefined = undefined;
+	includeNodes = config.getEnv('nodes.include');
 
 	logger: ILogger;
 
 	async init() {
-		this.logger = getLogger();
-		LoggerProxy.init(this.logger);
-
 		// Make sure the imported modules can resolve dependencies fine.
 		const delimiter = process.platform === 'win32' ? ';' : ':';
 		process.env.NODE_PATH = module.paths.join(delimiter);
+
 		// @ts-ignore
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-call
 		module.constructor._initPaths();
 
-		const nodeModulesPath = await this.getNodeModulesFolderLocation();
-
-		this.excludeNodes = config.getEnv('nodes.exclude');
-		this.includeNodes = config.getEnv('nodes.include');
-
-		// Get all the installed packages which contain n8n nodes
-		const nodePackages = await this.getN8nNodePackages(nodeModulesPath);
-
-		for (const packagePath of nodePackages) {
-			await this.loadDataFromPackage(packagePath);
-		}
+		await mkdir(path.join(GENERATED_STATIC_DIR, 'icons/nodes'), { recursive: true });
+		await mkdir(path.join(GENERATED_STATIC_DIR, 'icons/credentials'), { recursive: true });
 
+		await this.loadNodesFromBasePackages();
 		await this.loadNodesFromDownloadedPackages();
-
-		await this.loadNodesFromCustomFolders();
+		await this.loadNodesFromCustomDirectories();
 	}
 
-	async getNodeModulesFolderLocation(): Promise<string> {
-		// Get the path to the node-modules folder to be later able
-		// to load the credentials and nodes
-		const checkPaths = [
-			// In case "n8n" package is in same node_modules folder.
-			path.join(CLI_DIR, '..', 'n8n-workflow'),
-			// In case "n8n" package is the root and the packages are
-			// in the "node_modules" folder underneath it.
-			path.join(CLI_DIR, 'node_modules', 'n8n-workflow'),
-			// In case "n8n" package is installed using npm/yarn workspaces
-			// the node_modules folder is in the root of the workspace.
-			path.join(CLI_DIR, '..', '..', 'node_modules', 'n8n-workflow'),
-		];
-		for (const checkPath of checkPaths) {
-			try {
-				await fsAccess(checkPath);
-				// Folder exists, so use it.
-				return path.dirname(checkPath);
-			} catch (_) {
-				// Folder does not exist so get next one
+	async generateTypesForFrontend() {
+		const credentialsOverwrites = CredentialsOverwrites().getAll();
+		for (const credential of this.types.credentials) {
+			if (credential.name in credentialsOverwrites) {
+				credential.__overwrittenProperties = Object.keys(credentialsOverwrites[credential.name]);
 			}
 		}
-		throw new Error('Could not find "node_modules" folder!');
+
+		// pre-render all the node and credential types as static json files
+		await mkdir(path.join(GENERATED_STATIC_DIR, 'types'), { recursive: true });
+
+		const writeStaticJSON = async (name: string, data: any[]) => {
+			const filePath = path.join(GENERATED_STATIC_DIR, `types/${name}.json`);
+			const payload = `[\n${data.map((entry) => JSON.stringify(entry)).join(',\n')}\n]`;
+			await writeFile(filePath, payload, { encoding: 'utf-8' });
+		};
+
+		await writeStaticJSON('nodes', this.types.nodes);
+		await writeStaticJSON('credentials', this.types.credentials);
+	}
+
+	async loadNodesFromBasePackages() {
+		const nodeModulesPath = await this.getNodeModulesPath();
+		const nodePackagePaths = await this.getN8nNodePackages(nodeModulesPath);
+
+		for (const packagePath of nodePackagePaths) {
+			await this.runDirectoryLoader(LazyPackageDirectoryLoader, packagePath);
+		}
 	}
 
 	async loadNodesFromDownloadedPackages(): Promise<void> {
 		const nodePackages = [];
 		try {
 			// Read downloaded nodes and credentials
-			const downloadedNodesFolder = UserSettings.getUserN8nFolderDownloadedNodesPath();
-			const downloadedNodesFolderModules = path.join(downloadedNodesFolder, 'node_modules');
-			await fsAccess(downloadedNodesFolderModules);
-			const downloadedPackages = await this.getN8nNodePackages(downloadedNodesFolderModules);
+			const downloadedNodesDir = UserSettings.getUserN8nFolderDownloadedNodesPath();
+			const downloadedNodesDirModules = path.join(downloadedNodesDir, 'node_modules');
+			await fsAccess(downloadedNodesDirModules);
+			const downloadedPackages = await this.getN8nNodePackages(downloadedNodesDirModules);
 			nodePackages.push(...downloadedPackages);
 		} catch (error) {
 			// Folder does not exist so ignore and return
@@ -135,15 +111,14 @@ class LoadNodesAndCredentialsClass {
 
 		for (const packagePath of nodePackages) {
 			try {
-				await this.loadDataFromPackage(packagePath);
-				// eslint-disable-next-line no-empty
+				await this.runDirectoryLoader(PackageDirectoryLoader, packagePath);
 			} catch (error) {
 				ErrorReporter.error(error);
 			}
 		}
 	}
 
-	async loadNodesFromCustomFolders(): Promise<void> {
+	async loadNodesFromCustomDirectories(): Promise<void> {
 		// Read nodes and credentials from custom directories
 		const customDirectories = [];
 
@@ -158,7 +133,7 @@ class LoadNodesAndCredentialsClass {
 		}
 
 		for (const directory of customDirectories) {
-			await this.loadDataFromDirectory('CUSTOM', directory);
+			await this.runDirectoryLoader(CustomDirectoryLoader, directory);
 		}
 	}
 
@@ -192,46 +167,6 @@ class LoadNodesAndCredentialsClass {
 		return getN8nNodePackagesRecursive('');
 	}
 
-	/**
-	 * Loads credentials from a file
-	 *
-	 * @param {string} credentialName The name of the credentials
-	 * @param {string} filePath The file to read credentials from
-	 */
-	loadCredentialsFromFile(credentialName: string, filePath: string): void {
-		let tempCredential: ICredentialType;
-		try {
-			tempCredential = loadClassInIsolation(filePath, credentialName);
-
-			// Add serializer method "toJSON" to the class so that authenticate method (if defined)
-			// gets mapped to the authenticate attribute before it is sent to the client.
-			// The authenticate property is used by the client to decide whether or not to
-			// include the credential type in the predefined credentials (HTTP node)
-			Object.assign(tempCredential, { toJSON });
-
-			if (tempCredential.icon && tempCredential.icon.startsWith('file:')) {
-				// If a file icon gets used add the full path
-				tempCredential.icon = `file:${path.join(
-					path.dirname(filePath),
-					tempCredential.icon.substr(5),
-				)}`;
-			}
-		} catch (e) {
-			if (e instanceof TypeError) {
-				throw new Error(
-					`Class with name "${credentialName}" could not be found. Please check if the class is named correctly!`,
-				);
-			} else {
-				throw e;
-			}
-		}
-
-		this.credentialTypes[tempCredential.name] = {
-			type: tempCredential,
-			sourcePath: filePath,
-		};
-	}
-
 	async loadNpmModule(packageName: string, version?: string): Promise<InstalledPackages> {
 		const downloadFolder = UserSettings.getUserN8nFolderDownloadedNodesPath();
 		const command = `npm install ${packageName}${version ? `@${version}` : ''}`;
@@ -240,24 +175,30 @@ class LoadNodesAndCredentialsClass {
 
 		const finalNodeUnpackedPath = path.join(downloadFolder, 'node_modules', packageName);
 
-		const loadedNodes = await this.loadDataFromPackage(finalNodeUnpackedPath);
+		const { loadedNodes, packageJson } = await this.runDirectoryLoader(
+			PackageDirectoryLoader,
+			finalNodeUnpackedPath,
+		);
 
 		if (loadedNodes.length > 0) {
-			const packageFile = await this.readPackageJson(finalNodeUnpackedPath);
 			// Save info to DB
 			try {
 				const installedPackage = await persistInstalledPackageData(
-					packageFile.name,
-					packageFile.version,
+					packageJson.name,
+					packageJson.version,
 					loadedNodes,
-					this.nodeTypes,
-					packageFile.author?.name,
-					packageFile.author?.email,
+					this.loaded.nodes,
+					packageJson.author?.name,
+					packageJson.author?.email,
 				);
 				this.attachNodesToNodeTypes(installedPackage.installedNodes);
+				await this.generateTypesForFrontend();
 				return installedPackage;
 			} catch (error) {
-				LoggerProxy.error('Failed to save installed packages and nodes', { error, packageName });
+				LoggerProxy.error('Failed to save installed packages and nodes', {
+					error: error as Error,
+					packageName,
+				});
 				throw error;
 			}
 		} else {
@@ -265,9 +206,7 @@ class LoadNodesAndCredentialsClass {
 			const removeCommand = `npm remove ${packageName}`;
 			try {
 				await executeCommand(removeCommand);
-			} catch (error) {
-				// Do nothing
-			}
+			} catch (_) {}
 
 			throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_DOES_NOT_CONTAIN_NODES);
 		}
@@ -278,7 +217,9 @@ class LoadNodesAndCredentialsClass {
 
 		await executeCommand(command);
 
-		void (await removePackageFromDatabase(installedPackage));
+		await removePackageFromDatabase(installedPackage);
+
+		await this.generateTypesForFrontend();
 
 		this.unloadNodes(installedPackage.installedNodes);
 	}
@@ -294,7 +235,7 @@ class LoadNodesAndCredentialsClass {
 		try {
 			await executeCommand(command);
 		} catch (error) {
-			if (error.message === RESPONSE_ERROR_MESSAGES.PACKAGE_NOT_FOUND) {
+			if (error instanceof Error && error.message === RESPONSE_ERROR_MESSAGES.PACKAGE_NOT_FOUND) {
 				throw new Error(`The npm package "${packageName}" could not be found.`);
 			}
 			throw error;
@@ -304,29 +245,35 @@ class LoadNodesAndCredentialsClass {
 
 		const finalNodeUnpackedPath = path.join(downloadFolder, 'node_modules', packageName);
 
-		const loadedNodes = await this.loadDataFromPackage(finalNodeUnpackedPath);
+		const { loadedNodes, packageJson } = await this.runDirectoryLoader(
+			PackageDirectoryLoader,
+			finalNodeUnpackedPath,
+		);
 
 		if (loadedNodes.length > 0) {
-			const packageFile = await this.readPackageJson(finalNodeUnpackedPath);
-
 			// Save info to DB
 			try {
 				await removePackageFromDatabase(installedPackage);
 
 				const newlyInstalledPackage = await persistInstalledPackageData(
-					packageFile.name,
-					packageFile.version,
+					packageJson.name,
+					packageJson.version,
 					loadedNodes,
-					this.nodeTypes,
-					packageFile.author?.name,
-					packageFile.author?.email,
+					this.loaded.nodes,
+					packageJson.author?.name,
+					packageJson.author?.email,
 				);
 
 				this.attachNodesToNodeTypes(newlyInstalledPackage.installedNodes);
 
+				await this.generateTypesForFrontend();
+
 				return newlyInstalledPackage;
 			} catch (error) {
-				LoggerProxy.error('Failed to save installed packages and nodes', { error, packageName });
+				LoggerProxy.error('Failed to save installed packages and nodes', {
+					error: error as Error,
+					packageName,
+				});
 				throw error;
 			}
 		} else {
@@ -334,249 +281,119 @@ class LoadNodesAndCredentialsClass {
 			const removeCommand = `npm remove ${packageName}`;
 			try {
 				await executeCommand(removeCommand);
-			} catch (error) {
-				// Do nothing
-			}
+			} catch (_) {}
 			throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_DOES_NOT_CONTAIN_NODES);
 		}
 	}
 
-	/**
-	 * Loads a node from a file
-	 *
-	 * @param {string} packageName The package name to set for the found nodes
-	 * @param {string} nodeName Tha name of the node
-	 * @param {string} filePath The file to read node from
-	 */
-	loadNodeFromFile(
-		packageName: string,
-		nodeName: string,
-		filePath: string,
-	): INodeTypeNameVersion | undefined {
-		let tempNode: INodeType | IVersionedNodeType;
-		let nodeVersion = 1;
+	private unloadNodes(installedNodes: InstalledNodes[]): void {
+		installedNodes.forEach((installedNode) => {
+			delete this.loaded.nodes[installedNode.type];
+		});
+	}
 
-		try {
-			tempNode = loadClassInIsolation(filePath, nodeName);
-			this.addCodex({ node: tempNode, filePath, isCustom: packageName === 'CUSTOM' });
-		} catch (error) {
-			// eslint-disable-next-line no-console, @typescript-eslint/restrict-template-expressions
-			console.error(`Error loading node "${nodeName}" from: "${filePath}" - ${error.message}`);
-			throw error;
-		}
-
-		const fullNodeName = `${packageName}.${tempNode.description.name}`;
-		tempNode.description.name = fullNodeName;
-
-		if (tempNode.description.icon !== undefined && tempNode.description.icon.startsWith('file:')) {
-			// If a file icon gets used add the full path
-			tempNode.description.icon = `file:${path.join(
-				path.dirname(filePath),
-				tempNode.description.icon.substr(5),
-			)}`;
-		}
-
-		if (tempNode.hasOwnProperty('nodeVersions')) {
-			const versionedNodeType = (tempNode as IVersionedNodeType).getNodeType();
-			this.addCodex({ node: versionedNodeType, filePath, isCustom: packageName === 'CUSTOM' });
-			nodeVersion = (tempNode as IVersionedNodeType).currentVersion;
-
-			if (
-				versionedNodeType.description.icon !== undefined &&
-				versionedNodeType.description.icon.startsWith('file:')
-			) {
-				// If a file icon gets used add the full path
-				versionedNodeType.description.icon = `file:${path.join(
-					path.dirname(filePath),
-					versionedNodeType.description.icon.substr(5),
-				)}`;
-			}
-
-			if (versionedNodeType.hasOwnProperty('executeSingle')) {
-				this.logger.warn(
-					`"executeSingle" will get deprecated soon. Please update the code of node "${packageName}.${nodeName}" to use "execute" instead!`,
-					{ filePath },
-				);
-			}
-		} else {
-			// Short renaming to avoid type issues
-			const tmpNode = tempNode as INodeType;
-			nodeVersion = Array.isArray(tmpNode.description.version)
-				? tmpNode.description.version.slice(-1)[0]
-				: tmpNode.description.version;
-		}
-
-		if (this.includeNodes !== undefined && !this.includeNodes.includes(fullNodeName)) {
-			return;
-		}
-
-		// Check if the node should be skipped
-		if (this.excludeNodes !== undefined && this.excludeNodes.includes(fullNodeName)) {
-			return;
-		}
-
-		this.nodeTypes[fullNodeName] = {
-			type: tempNode,
-			sourcePath: filePath,
-		};
-
-		// eslint-disable-next-line consistent-return
-		return {
-			name: fullNodeName,
-			version: nodeVersion,
-		} as INodeTypeNameVersion;
+	private attachNodesToNodeTypes(installedNodes: InstalledNodes[]): void {
+		const loadedNodes = this.loaded.nodes;
+		installedNodes.forEach((installedNode) => {
+			const { type, sourcePath } = loadedNodes[installedNode.type];
+			loadedNodes[installedNode.type] = { type, sourcePath };
+		});
 	}
 
 	/**
-	 * Retrieves `categories`, `subcategories`, partial `resources` and
-	 * alias (if defined) from the codex data for the node at the given file path.
-	 *
-	 * @param {string} filePath The file path to a `*.node.js` file
+	 * Run a loader of source files of nodes and credentials in a directory.
 	 */
-	getCodex(filePath: string): CodexData {
-		// eslint-disable-next-line global-require, import/no-dynamic-require, @typescript-eslint/no-var-requires
-		const { categories, subcategories, resources: allResources, alias } = require(`${filePath}on`); // .js to .json
+	private async runDirectoryLoader<T extends DirectoryLoader>(
+		constructor: new (...args: ConstructorParameters<typeof DirectoryLoader>) => T,
+		dir: string,
+	) {
+		const loader = new constructor(dir, this.excludeNodes, this.includeNodes);
+		await loader.loadAll();
 
-		const resources = pick(allResources, ['primaryDocumentation', 'credentialDocumentation']);
+		// list of node & credential types that will be sent to the frontend
+		const { types } = loader;
+		this.types.nodes = this.types.nodes.concat(types.nodes);
+		this.types.credentials = this.types.credentials.concat(types.credentials);
 
-		// eslint-disable-next-line @typescript-eslint/no-unsafe-return
-		return {
-			...(categories && { categories }),
-			...(subcategories && { subcategories }),
-			...(resources && { resources }),
-			...(alias && { alias }),
-		};
-	}
-
-	/**
-	 * Adds a node codex `categories` and `subcategories` (if defined)
-	 * to a node description `codex` property.
-	 *
-	 * @param obj.node Node to add categories to
-	 * @param obj.filePath Path to the built node
-	 * @param obj.isCustom Whether the node is custom
-	 */
-	addCodex({
-		node,
-		filePath,
-		isCustom,
-	}: {
-		node: INodeType | IVersionedNodeType;
-		filePath: string;
-		isCustom: boolean;
-	}) {
-		try {
-			const codex = this.getCodex(filePath);
-
-			if (isCustom) {
-				codex.categories = codex.categories
-					? codex.categories.concat(CUSTOM_NODES_CATEGORY)
-					: [CUSTOM_NODES_CATEGORY];
+		// Copy over all icons and set `iconUrl` for the frontend
+		const iconPromises: Array<Promise<void>> = [];
+		for (const node of types.nodes) {
+			if (node.icon?.startsWith('file:')) {
+				const icon = node.icon.substring(5);
+				const iconUrl = `icons/nodes/${node.name}${path.extname(icon)}`;
+				delete node.icon;
+				node.iconUrl = iconUrl;
+				iconPromises.push(copyFile(path.join(dir, icon), path.join(GENERATED_STATIC_DIR, iconUrl)));
 			}
+		}
+		for (const credential of types.credentials) {
+			if (credential.icon?.startsWith('file:')) {
+				const icon = credential.icon.substring(5);
+				const iconUrl = `icons/credentials/${credential.name}${path.extname(icon)}`;
+				delete credential.icon;
+				credential.iconUrl = iconUrl;
+				iconPromises.push(copyFile(path.join(dir, icon), path.join(GENERATED_STATIC_DIR, iconUrl)));
+			}
+		}
+		await Promise.all(iconPromises);
 
-			node.description.codex = codex;
-		} catch (_) {
-			this.logger.debug(`No codex available for: ${filePath.split('/').pop() ?? ''}`);
+		// Nodes and credentials that have been loaded immediately
+		for (const nodeTypeName in loader.nodeTypes) {
+			this.loaded.nodes[nodeTypeName] = loader.nodeTypes[nodeTypeName];
+		}
 
-			if (isCustom) {
-				node.description.codex = {
-					categories: [CUSTOM_NODES_CATEGORY],
+		for (const credentialTypeName in loader.credentialTypes) {
+			this.loaded.credentials[credentialTypeName] = loader.credentialTypes[credentialTypeName];
+		}
+
+		// Nodes and credentials that will be lazy loaded
+		if (loader instanceof LazyPackageDirectoryLoader) {
+			const { packageName, known } = loader;
+
+			for (const type in known.nodes) {
+				const { className, sourcePath } = known.nodes[type];
+				this.known.nodes[`${packageName}.${type}`] = {
+					className,
+					sourcePath: path.join(dir, sourcePath),
 				};
 			}
-		}
-	}
 
-	/**
-	 * Loads nodes and credentials from the given directory
-	 *
-	 * @param {string} setPackageName The package name to set for the found nodes
-	 * @param {string} directory The directory to look in
-	 */
-	async loadDataFromDirectory(setPackageName: string, directory: string): Promise<void> {
-		const files = await glob('**/*.@(node|credentials).js', {
-			cwd: directory,
-			absolute: true,
-		});
-
-		for (const filePath of files) {
-			const [fileName, type] = path.parse(filePath).name.split('.');
-
-			if (type === 'node') {
-				this.loadNodeFromFile(setPackageName, fileName, filePath);
-			} else if (type === 'credentials') {
-				this.loadCredentialsFromFile(fileName, filePath);
-			}
-		}
-	}
-
-	async readPackageJson(packagePath: string): Promise<IN8nNodePackageJson> {
-		// Get the absolute path of the package
-		const packageFileString = await fsReadFile(path.join(packagePath, 'package.json'), 'utf8');
-		return jsonParse(packageFileString);
-	}
-
-	/**
-	 * Loads nodes and credentials from the package with the given name
-	 *
-	 * @param {string} packagePath The path to read data from
-	 */
-	async loadDataFromPackage(packagePath: string): Promise<INodeTypeNameVersion[]> {
-		// Get the absolute path of the package
-		const packageFile = await this.readPackageJson(packagePath);
-		if (!packageFile.n8n) {
-			return [];
-		}
-
-		const packageName = packageFile.name;
-		const { nodes, credentials } = packageFile.n8n;
-		const returnData: INodeTypeNameVersion[] = [];
-
-		// Read all node types
-		if (Array.isArray(nodes)) {
-			for (const filePath of nodes) {
-				const tempPath = path.join(packagePath, filePath);
-				const [fileName] = path.parse(filePath).name.split('.');
-				const loadData = this.loadNodeFromFile(packageName, fileName, tempPath);
-				if (loadData) {
-					returnData.push(loadData);
-				}
+			for (const type in known.credentials) {
+				const { className, sourcePath } = known.credentials[type];
+				this.known.credentials[type] = { className, sourcePath: path.join(dir, sourcePath) };
 			}
 		}
 
-		// Read all credential types
-		if (Array.isArray(credentials)) {
-			for (const filePath of credentials) {
-				const tempPath = path.join(packagePath, filePath);
-				const [fileName] = path.parse(filePath).name.split('.');
-				this.loadCredentialsFromFile(fileName, tempPath);
-			}
+		return loader;
+	}
+
+	private async getNodeModulesPath(): Promise<string> {
+		// Get the path to the node-modules folder to be later able
+		// to load the credentials and nodes
+		const checkPaths = [
+			// In case "n8n" package is in same node_modules folder.
+			path.join(CLI_DIR, '..', 'n8n-workflow'),
+			// In case "n8n" package is the root and the packages are
+			// in the "node_modules" folder underneath it.
+			path.join(CLI_DIR, 'node_modules', 'n8n-workflow'),
+			// In case "n8n" package is installed using npm/yarn workspaces
+			// the node_modules folder is in the root of the workspace.
+			path.join(CLI_DIR, '..', '..', 'node_modules', 'n8n-workflow'),
+		];
+		for (const checkPath of checkPaths) {
+			try {
+				await fsAccess(checkPath);
+				// Folder exists, so use it.
+				return path.dirname(checkPath);
+			} catch (_) {} // Folder does not exist so get next one
 		}
-
-		return returnData;
-	}
-
-	unloadNodes(installedNodes: InstalledNodes[]): void {
-		const nodeTypes = NodeTypes();
-		installedNodes.forEach((installedNode) => {
-			nodeTypes.removeNodeType(installedNode.type);
-			delete this.nodeTypes[installedNode.type];
-		});
-	}
-
-	attachNodesToNodeTypes(installedNodes: InstalledNodes[]): void {
-		const nodeTypes = NodeTypes();
-		installedNodes.forEach((installedNode) => {
-			nodeTypes.attachNodeType(
-				installedNode.type,
-				this.nodeTypes[installedNode.type].type,
-				this.nodeTypes[installedNode.type].sourcePath,
-			);
-		});
+		throw new Error('Could not find "node_modules" folder!');
 	}
 }
 
 let packagesInformationInstance: LoadNodesAndCredentialsClass | undefined;
 
+// eslint-disable-next-line @typescript-eslint/naming-convention
 export function LoadNodesAndCredentials(): LoadNodesAndCredentialsClass {
 	if (packagesInformationInstance === undefined) {
 		packagesInformationInstance = new LoadNodesAndCredentialsClass();

packages/cli/src/NodeTypes.ts

@@ -1,36 +1,28 @@
-/* eslint-disable @typescript-eslint/no-unsafe-return */
-/* eslint-disable @typescript-eslint/no-unsafe-call */
-/* eslint-disable @typescript-eslint/no-unsafe-member-access */
-/* eslint-disable @typescript-eslint/no-unsafe-assignment */
-import {
+import { loadClassInIsolation } from 'n8n-core';
+import type {
+	INodesAndCredentials,
 	INodeType,
-	INodeTypeData,
 	INodeTypeDescription,
 	INodeTypes,
 	IVersionedNodeType,
-	NodeHelpers,
+	LoadedClass,
 } from 'n8n-workflow';
+import { NodeHelpers } from 'n8n-workflow';
+import { RESPONSE_ERROR_MESSAGES } from './constants';
 
 class NodeTypesClass implements INodeTypes {
-	nodeTypes: INodeTypeData = {};
-
-	async init(nodeTypes: INodeTypeData): Promise<void> {
+	constructor(private nodesAndCredentials: INodesAndCredentials) {
 		// Some nodeTypes need to get special parameters applied like the
 		// polling nodes the polling times
 		// eslint-disable-next-line no-restricted-syntax
-		for (const nodeTypeData of Object.values(nodeTypes)) {
+		for (const nodeTypeData of Object.values(this.loadedNodes)) {
 			const nodeType = NodeHelpers.getVersionedNodeType(nodeTypeData.type);
-			const applyParameters = NodeHelpers.getSpecialNodeParameters(nodeType);
-
-			if (applyParameters.length) {
-				nodeType.description.properties.unshift(...applyParameters);
-			}
+			this.applySpecialNodeParameters(nodeType);
 		}
-		this.nodeTypes = nodeTypes;
 	}
 
 	getAll(): Array<INodeType | IVersionedNodeType> {
-		return Object.values(this.nodeTypes).map((data) => data.type);
+		return Object.values(this.loadedNodes).map(({ type }) => type);
 	}
 
 	/**
@@ -40,7 +32,7 @@ class NodeTypesClass implements INodeTypes {
 		nodeTypeName: string,
 		version: number,
 	): { description: INodeTypeDescription } & { sourcePath: string } {
-		const nodeType = this.nodeTypes[nodeTypeName];
+		const nodeType = this.getNode(nodeTypeName);
 
 		if (!nodeType) {
 			throw new Error(`Unknown node type: ${nodeTypeName}`);
@@ -52,34 +44,52 @@ class NodeTypesClass implements INodeTypes {
 	}
 
 	getByNameAndVersion(nodeType: string, version?: number): INodeType {
-		if (this.nodeTypes[nodeType] === undefined) {
-			throw new Error(`The node-type "${nodeType}" is not known!`);
+		return NodeHelpers.getVersionedNodeType(this.getNode(nodeType).type, version);
+	}
+
+	private getNode(type: string): LoadedClass<INodeType | IVersionedNodeType> {
+		const loadedNodes = this.loadedNodes;
+		if (type in loadedNodes) {
+			return loadedNodes[type];
 		}
-		return NodeHelpers.getVersionedNodeType(this.nodeTypes[nodeType].type, version);
+
+		const knownNodes = this.knownNodes;
+		if (type in knownNodes) {
+			const { className, sourcePath } = knownNodes[type];
+			const loaded: INodeType = loadClassInIsolation(sourcePath, className);
+			this.applySpecialNodeParameters(loaded);
+			loadedNodes[type] = { sourcePath, type: loaded };
+			return loadedNodes[type];
+		}
+		throw new Error(`${RESPONSE_ERROR_MESSAGES.NO_NODE}: ${type}`);
 	}
 
-	attachNodeType(
-		nodeTypeName: string,
-		nodeType: INodeType | IVersionedNodeType,
-		sourcePath: string,
-	): void {
-		this.nodeTypes[nodeTypeName] = {
-			type: nodeType,
-			sourcePath,
-		};
+	private applySpecialNodeParameters(nodeType: INodeType) {
+		const applyParameters = NodeHelpers.getSpecialNodeParameters(nodeType);
+		if (applyParameters.length) {
+			nodeType.description.properties.unshift(...applyParameters);
+		}
 	}
 
-	removeNodeType(nodeType: string): void {
-		delete this.nodeTypes[nodeType];
+	private get loadedNodes() {
+		return this.nodesAndCredentials.loaded.nodes;
+	}
+
+	private get knownNodes() {
+		return this.nodesAndCredentials.known.nodes;
 	}
 }
 
 let nodeTypesInstance: NodeTypesClass | undefined;
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
-export function NodeTypes(): NodeTypesClass {
-	if (nodeTypesInstance === undefined) {
-		nodeTypesInstance = new NodeTypesClass();
+export function NodeTypes(nodesAndCredentials?: INodesAndCredentials): NodeTypesClass {
+	if (!nodeTypesInstance) {
+		if (nodesAndCredentials) {
+			nodeTypesInstance = new NodeTypesClass(nodesAndCredentials);
+		} else {
+			throw new Error('NodeTypes not initialized yet');
+		}
 	}
 
 	return nodeTypesInstance;

packages/cli/src/Server.ts

@@ -9,7 +9,6 @@
 /* eslint-disable no-return-assign */
 /* eslint-disable no-param-reassign */
 /* eslint-disable consistent-return */
-/* eslint-disable import/no-extraneous-dependencies */
 /* eslint-disable @typescript-eslint/restrict-template-expressions */
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 /* eslint-disable id-denylist */
@@ -29,7 +28,7 @@
 /* eslint-disable no-await-in-loop */
 
 import { exec as callbackExec } from 'child_process';
-import { existsSync, readFileSync } from 'fs';
+import { existsSync, readFileSync, writeFileSync } from 'fs';
 import { access as fsAccess, readFile, writeFile, mkdir } from 'fs/promises';
 import os from 'os';
 import { dirname as pathDirname, join as pathJoin, resolve as pathResolve } from 'path';
@@ -38,7 +37,6 @@ import { promisify } from 'util';
 import cookieParser from 'cookie-parser';
 import express from 'express';
 import { FindManyOptions, getConnectionManager, In } from 'typeorm';
-// eslint-disable-next-line import/no-extraneous-dependencies
 import axios, { AxiosRequestConfig } from 'axios';
 import clientOAuth1, { RequestOptions } from 'oauth-1.0a';
 // IMPORTANT! Do not switch to anther bcrypt library unless really necessary and
@@ -54,22 +52,20 @@ import {
 } from 'n8n-core';
 
 import {
-	ICredentialType,
 	INodeCredentials,
 	INodeCredentialsDetails,
 	INodeListSearchResult,
 	INodeParameters,
 	INodePropertyOptions,
-	INodeType,
-	INodeTypeDescription,
 	INodeTypeNameVersion,
 	ITelemetrySettings,
 	LoggerProxy,
-	NodeHelpers,
 	jsonParse,
 	WebhookHttpMethod,
 	WorkflowExecuteMode,
 	ErrorReporterProxy as ErrorReporter,
+	INodeTypes,
+	ICredentialTypes,
 } from 'n8n-workflow';
 
 import basicAuth from 'basic-auth';
@@ -95,6 +91,7 @@ import { nodesController } from '@/api/nodes.api';
 import { workflowsController } from '@/workflows/workflows.controller';
 import {
 	AUTH_COOKIE_NAME,
+	GENERATED_STATIC_DIR,
 	NODES_BASE_DIR,
 	RESPONSE_ERROR_MESSAGES,
 	TEMPLATES_DIR,
@@ -151,6 +148,7 @@ import { ExternalHooks } from '@/ExternalHooks';
 import * as GenericHelpers from '@/GenericHelpers';
 import { NodeTypes } from '@/NodeTypes';
 import * as Push from '@/Push';
+import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
 import * as ResponseHelper from '@/ResponseHelper';
 import * as TestWebhooks from '@/TestWebhooks';
 import { WaitTracker, WaitTrackerClass } from '@/WaitTracker';
@@ -226,6 +224,10 @@ class App {
 
 	webhookMethods: WebhookHttpMethod[];
 
+	nodeTypes: INodeTypes;
+
+	credentialTypes: ICredentialTypes;
+
 	constructor() {
 		this.app = express();
 		this.app.disable('x-powered-by');
@@ -251,6 +253,9 @@ class App {
 		this.testWebhooks = TestWebhooks.getInstance();
 		this.push = Push.getInstance();
 
+		this.nodeTypes = NodeTypes();
+		this.credentialTypes = CredentialTypes();
+
 		this.activeExecutionsInstance = ActiveExecutions.getInstance();
 		this.waitTracker = WaitTracker();
 
@@ -424,6 +429,8 @@ class App {
 			'assets',
 			'healthz',
 			'metrics',
+			'icons',
+			'types',
 			this.endpointWebhook,
 			this.endpointWebhookTest,
 			this.endpointPresetCredentials,
@@ -824,7 +831,7 @@ class App {
 
 					const loadDataInstance = new LoadNodeParameterOptions(
 						nodeTypeAndVersion,
-						NodeTypes(),
+						this.nodeTypes,
 						path,
 						currentNodeParameters,
 						credentials,
@@ -885,7 +892,7 @@ class App {
 
 					const listSearchInstance = new LoadNodeListSearch(
 						nodeTypeAndVersion,
-						NodeTypes(),
+						this.nodeTypes,
 						path,
 						currentNodeParameters,
 						credentials,
@@ -910,47 +917,6 @@ class App {
 			),
 		);
 
-		// Returns all the node-types
-		this.app.get(
-			`/${this.restEndpoint}/node-types`,
-			ResponseHelper.send(
-				async (req: express.Request, res: express.Response): Promise<INodeTypeDescription[]> => {
-					const returnData: INodeTypeDescription[] = [];
-					const onlyLatest = req.query.onlyLatest === 'true';
-
-					const nodeTypes = NodeTypes();
-					const allNodes = nodeTypes.getAll();
-
-					const getNodeDescription = (nodeType: INodeType): INodeTypeDescription => {
-						const nodeInfo: INodeTypeDescription = { ...nodeType.description };
-						if (req.query.includeProperties !== 'true') {
-							// @ts-ignore
-							delete nodeInfo.properties;
-						}
-						return nodeInfo;
-					};
-
-					if (onlyLatest) {
-						allNodes.forEach((nodeData) => {
-							const nodeType = NodeHelpers.getVersionedNodeType(nodeData);
-							const nodeInfo: INodeTypeDescription = getNodeDescription(nodeType);
-							returnData.push(nodeInfo);
-						});
-					} else {
-						allNodes.forEach((nodeData) => {
-							const allNodeTypes = NodeHelpers.getVersionedNodeTypeAll(nodeData);
-							allNodeTypes.forEach((element) => {
-								const nodeInfo: INodeTypeDescription = getNodeDescription(element);
-								returnData.push(nodeInfo);
-							});
-						});
-					}
-
-					return returnData;
-				},
-			),
-		);
-
 		this.app.get(
 			`/${this.restEndpoint}/credential-translation`,
 			ResponseHelper.send(
@@ -999,49 +965,6 @@ class App {
 
 		this.app.use(`/${this.restEndpoint}/node-types`, nodeTypesController);
 
-		// Returns the node icon
-		this.app.get(
-			[
-				`/${this.restEndpoint}/node-icon/:nodeType`,
-				`/${this.restEndpoint}/node-icon/:scope/:nodeType`,
-			],
-			async (req: express.Request, res: express.Response): Promise<void> => {
-				try {
-					const nodeTypeName = `${req.params.scope ? `${req.params.scope}/` : ''}${
-						req.params.nodeType
-					}`;
-
-					const nodeTypes = NodeTypes();
-					const nodeType = nodeTypes.getByNameAndVersion(nodeTypeName);
-
-					if (nodeType === undefined) {
-						res.status(404).send('The nodeType is not known.');
-						return;
-					}
-
-					if (nodeType.description.icon === undefined) {
-						res.status(404).send('No icon found for node.');
-						return;
-					}
-
-					if (!nodeType.description.icon.startsWith('file:')) {
-						res.status(404).send('Node does not have a file icon.');
-						return;
-					}
-
-					const filepath = nodeType.description.icon.substr(5);
-
-					const maxAge = 7 * 24 * 60 * 60 * 1000; // 7 days
-					res.setHeader('Cache-control', `private max-age=${maxAge}`);
-
-					res.sendFile(filepath);
-				} catch (error) {
-					// Error response
-					return ResponseHelper.sendErrorResponse(res, error);
-				}
-			},
-		);
-
 		// ----------------------------------------
 		// Active Workflows
 		// ----------------------------------------
@@ -1107,63 +1030,6 @@ class App {
 				},
 			),
 		);
-		// ----------------------------------------
-		// Credential-Types
-		// ----------------------------------------
-
-		// Returns all the credential types which are defined in the loaded n8n-modules
-		this.app.get(
-			`/${this.restEndpoint}/credential-types`,
-			ResponseHelper.send(
-				async (req: express.Request, res: express.Response): Promise<ICredentialType[]> => {
-					const returnData: ICredentialType[] = [];
-
-					const credentialTypes = CredentialTypes();
-
-					credentialTypes.getAll().forEach((credentialData) => {
-						returnData.push(credentialData);
-					});
-
-					return returnData;
-				},
-			),
-		);
-
-		this.app.get(
-			`/${this.restEndpoint}/credential-icon/:credentialType`,
-			async (req: express.Request, res: express.Response): Promise<void> => {
-				try {
-					const credentialName = req.params.credentialType;
-
-					const credentialType = CredentialTypes().getByName(credentialName);
-
-					if (credentialType === undefined) {
-						res.status(404).send('The credentialType is not known.');
-						return;
-					}
-
-					if (credentialType.icon === undefined) {
-						res.status(404).send('No icon found for credential.');
-						return;
-					}
-
-					if (!credentialType.icon.startsWith('file:')) {
-						res.status(404).send('Credential does not have a file icon.');
-						return;
-					}
-
-					const filepath = credentialType.icon.substr(5);
-
-					const maxAge = 7 * 24 * 60 * 60 * 1000; // 7 days
-					res.setHeader('Cache-control', `private max-age=${maxAge}`);
-
-					res.sendFile(filepath);
-				} catch (error) {
-					// Error response
-					return ResponseHelper.sendErrorResponse(res, error);
-				}
-			},
-		);
 
 		// ----------------------------------------
 		// OAuth1-Credential/Auth
@@ -1750,9 +1616,9 @@ class App {
 							return;
 						}
 
-						const credentialsOverwrites = CredentialsOverwrites();
+						CredentialsOverwrites().setData(body);
 
-						await credentialsOverwrites.init(body);
+						await LoadNodesAndCredentials().generateTypesForFrontend();
 
 						this.presetCredentialsLoaded = true;
 
@@ -1792,7 +1658,6 @@ class App {
 			}
 
 			const editorUiDistDir = pathJoin(pathDirname(require.resolve('n8n-editor-ui')), 'dist');
-			const generatedStaticDir = pathJoin(UserSettings.getUserHome(), '.cache/n8n/public');
 
 			const closingTitleTag = '</title>';
 			const compileFile = async (fileName: string) => {
@@ -1805,7 +1670,7 @@ class App {
 					if (filePath.endsWith('index.html')) {
 						payload = payload.replace(closingTitleTag, closingTitleTag + scriptsString);
 					}
-					const destFile = pathJoin(generatedStaticDir, fileName);
+					const destFile = pathJoin(GENERATED_STATIC_DIR, fileName);
 					await mkdir(pathDirname(destFile), { recursive: true });
 					await writeFile(destFile, payload, 'utf-8');
 				}
@@ -1815,13 +1680,15 @@ class App {
 			const files = await glob('**/*.{css,js}', { cwd: editorUiDistDir });
 			await Promise.all(files.map(compileFile));
 
-			this.app.use('/', express.static(generatedStaticDir), express.static(editorUiDistDir));
+			this.app.use('/', express.static(GENERATED_STATIC_DIR), express.static(editorUiDistDir));
 
 			const startTime = new Date().toUTCString();
 			this.app.use('/index.html', (req, res, next) => {
 				res.setHeader('Last-Modified', startTime);
 				next();
 			});
+		} else {
+			this.app.use('/', express.static(GENERATED_STATIC_DIR));
 		}
 	}
 }

packages/cli/src/UserManagement/Interfaces.ts

@@ -1,5 +1,4 @@
 import { Application } from 'express';
-import { JwtFromRequestFunction } from 'passport-jwt';
 import type { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
 import type { IExternalHooksClass, IPersonalizationSurveyAnswers } from '@/Interfaces';
 
@@ -8,11 +7,6 @@ export interface JwtToken {
 	expiresIn: number;
 }
 
-export interface JwtOptions {
-	secretOrKey: string;
-	jwtFromRequest: JwtFromRequestFunction;
-}
-
 export interface JwtPayload {
 	id: string;
 	email: string | null;

packages/cli/src/WebhookHelpers.ts

@@ -14,8 +14,7 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 /* eslint-disable prefer-destructuring */
 import express from 'express';
-// eslint-disable-next-line import/no-extraneous-dependencies
-import { get } from 'lodash';
+import get from 'lodash.get';
 
 import { BINARY_ENCODING, BinaryDataManager, NodeExecuteFunctions } from 'n8n-core';
 

packages/cli/src/WebhookServer.ts

@@ -13,7 +13,6 @@ import { getConnectionManager } from 'typeorm';
 import bodyParser from 'body-parser';
 
 import compression from 'compression';
-// eslint-disable-next-line import/no-extraneous-dependencies
 import parseUrl from 'parseurl';
 import { WebhookHttpMethod } from 'n8n-workflow';
 

packages/cli/src/WorkflowExecuteAdditionalData.ts

@@ -880,10 +880,8 @@ export async function getWorkflowData(
 
 /**
  * Executes the workflow with the given ID
- *
- * @param {string} workflowId The id of the workflow to execute
  */
-export async function executeWorkflow(
+async function executeWorkflow(
 	workflowInfo: IExecuteWorkflowInfo,
 	additionalData: IWorkflowExecuteAdditionalData,
 	options?: {
@@ -1111,7 +1109,7 @@ export async function getBase(
  * Returns WorkflowHooks instance for running integrated workflows
  * (Workflows which get started inside of another workflow)
  */
-export function getWorkflowHooksIntegrated(
+function getWorkflowHooksIntegrated(
 	mode: WorkflowExecuteMode,
 	executionId: string,
 	workflowData: IWorkflowBase,

packages/cli/src/WorkflowHelpers.ts

@@ -1,13 +1,3 @@
-/* eslint-disable @typescript-eslint/no-unsafe-call */
-/* eslint-disable @typescript-eslint/no-unsafe-return */
-/* eslint-disable @typescript-eslint/no-unsafe-assignment */
-/* eslint-disable @typescript-eslint/no-non-null-assertion */
-/* eslint-disable no-underscore-dangle */
-/* eslint-disable no-continue */
-/* eslint-disable @typescript-eslint/no-unsafe-member-access */
-/* eslint-disable @typescript-eslint/restrict-template-expressions */
-/* eslint-disable no-restricted-syntax */
-/* eslint-disable no-param-reassign */
 import { In } from 'typeorm';
 import {
 	IDataObject,
@@ -25,15 +15,8 @@ import {
 	WorkflowExecuteMode,
 } from 'n8n-workflow';
 import { v4 as uuid } from 'uuid';
-import { CredentialTypes } from '@/CredentialTypes';
 import * as Db from '@/Db';
-import {
-	ICredentialsDb,
-	ICredentialsTypeData,
-	ITransferNodeTypes,
-	IWorkflowErrorData,
-	IWorkflowExecutionDataProcess,
-} from '@/Interfaces';
+import { ICredentialsDb, IWorkflowErrorData, IWorkflowExecutionDataProcess } from '@/Interfaces';
 import { NodeTypes } from '@/NodeTypes';
 import { WorkflowRunner } from '@/WorkflowRunner';
 
@@ -183,6 +166,7 @@ export async function executeErrorWorkflow(
 
 		if (workflowStartNode === undefined) {
 			Logger.error(
+				// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
 				`Calling Error Workflow for "${workflowErrorData.workflow.id}". Could not find "${ERROR_TRIGGER_TYPE}" in workflow "${workflowId}"`,
 			);
 			return;
@@ -231,170 +215,15 @@ export async function executeErrorWorkflow(
 	} catch (error) {
 		ErrorReporter.error(error);
 		Logger.error(
+			// eslint-disable-next-line @typescript-eslint/restrict-template-expressions, @typescript-eslint/no-unsafe-member-access
 			`Calling Error Workflow for "${workflowErrorData.workflow.id}": "${error.message}"`,
 			{ workflowId: workflowErrorData.workflow.id },
 		);
 	}
 }
 
-/**
- * Returns all the defined NodeTypes
- *
- */
-export function getAllNodeTypeData(): ITransferNodeTypes {
-	const nodeTypes = NodeTypes();
-
-	// Get the data of all the node types that they
-	// can be loaded again in the process
-	const returnData: ITransferNodeTypes = {};
-	for (const nodeTypeName of Object.keys(nodeTypes.nodeTypes)) {
-		if (nodeTypes.nodeTypes[nodeTypeName] === undefined) {
-			throw new Error(`The NodeType "${nodeTypeName}" could not be found!`);
-		}
-
-		returnData[nodeTypeName] = {
-			className: nodeTypes.nodeTypes[nodeTypeName].type.constructor.name,
-			sourcePath: nodeTypes.nodeTypes[nodeTypeName].sourcePath,
-		};
-	}
-
-	return returnData;
-}
-
-/**
- * Returns all the defined CredentialTypes
- *
- */
-export function getAllCredentalsTypeData(): ICredentialsTypeData {
-	const credentialTypes = CredentialTypes();
-
-	// Get the data of all the credential types that they
-	// can be loaded again in the subprocess
-	const returnData: ICredentialsTypeData = {};
-	for (const credentialTypeName of Object.keys(credentialTypes.credentialTypes)) {
-		if (credentialTypes.credentialTypes[credentialTypeName] === undefined) {
-			throw new Error(`The CredentialType "${credentialTypeName}" could not be found!`);
-		}
-
-		returnData[credentialTypeName] = {
-			className: credentialTypes.credentialTypes[credentialTypeName].type.constructor.name,
-			sourcePath: credentialTypes.credentialTypes[credentialTypeName].sourcePath,
-		};
-	}
-
-	return returnData;
-}
-
-/**
- * Returns the data of the node types that are needed
- * to execute the given nodes
- *
- */
-export function getNodeTypeData(nodes: INode[]): ITransferNodeTypes {
-	const nodeTypes = NodeTypes();
-
-	// Check which node-types have to be loaded
-	// eslint-disable-next-line @typescript-eslint/no-use-before-define
-	const neededNodeTypes = getNeededNodeTypes(nodes);
-
-	// Get all the data of the needed node types that they
-	// can be loaded again in the process
-	const returnData: ITransferNodeTypes = {};
-	for (const nodeTypeName of neededNodeTypes) {
-		if (nodeTypes.nodeTypes[nodeTypeName.type] === undefined) {
-			throw new Error(`The NodeType "${nodeTypeName.type}" could not be found!`);
-		}
-
-		returnData[nodeTypeName.type] = {
-			className: nodeTypes.nodeTypes[nodeTypeName.type].type.constructor.name,
-			sourcePath: nodeTypes.nodeTypes[nodeTypeName.type].sourcePath,
-		};
-	}
-
-	return returnData;
-}
-
-/**
- * Returns the credentials data of the given type and its parent types
- * it extends
- *
- * @param {string} type The credential type to return data off
- */
-export function getCredentialsDataWithParents(type: string): ICredentialsTypeData {
-	const credentialTypes = CredentialTypes();
-	const credentialType = credentialTypes.getByName(type);
-
-	const credentialTypeData: ICredentialsTypeData = {};
-	credentialTypeData[type] = {
-		className: credentialTypes.credentialTypes[type].type.constructor.name,
-		sourcePath: credentialTypes.credentialTypes[type].sourcePath,
-	};
-
-	if (credentialType === undefined || credentialType.extends === undefined) {
-		return credentialTypeData;
-	}
-
-	for (const typeName of credentialType.extends) {
-		if (credentialTypeData[typeName] !== undefined) {
-			continue;
-		}
-
-		credentialTypeData[typeName] = {
-			className: credentialTypes.credentialTypes[typeName].type.constructor.name,
-			sourcePath: credentialTypes.credentialTypes[typeName].sourcePath,
-		};
-		Object.assign(credentialTypeData, getCredentialsDataWithParents(typeName));
-	}
-
-	return credentialTypeData;
-}
-
-/**
- * Returns all the credentialTypes which are needed to resolve
- * the given workflow credentials
- *
- * @param {IWorkflowCredentials} credentials The credentials which have to be able to be resolved
- */
-export function getCredentialsDataByNodes(nodes: INode[]): ICredentialsTypeData {
-	const credentialTypeData: ICredentialsTypeData = {};
-
-	for (const node of nodes) {
-		const credentialsUsedByThisNode = node.credentials;
-		if (credentialsUsedByThisNode) {
-			// const credentialTypesUsedByThisNode = Object.keys(credentialsUsedByThisNode!);
-			for (const credentialType of Object.keys(credentialsUsedByThisNode)) {
-				if (credentialTypeData[credentialType] !== undefined) {
-					continue;
-				}
-
-				Object.assign(credentialTypeData, getCredentialsDataWithParents(credentialType));
-			}
-		}
-	}
-
-	return credentialTypeData;
-}
-
-/**
- * Returns the names of the NodeTypes which are are needed
- * to execute the gives nodes
- *
- */
-export function getNeededNodeTypes(nodes: INode[]): Array<{ type: string; version: number }> {
-	// Check which node-types have to be loaded
-	const neededNodeTypes: Array<{ type: string; version: number }> = [];
-	for (const node of nodes) {
-		if (neededNodeTypes.find((neededNodes) => node.type === neededNodes.type) === undefined) {
-			neededNodeTypes.push({ type: node.type, version: node.typeVersion });
-		}
-	}
-
-	return neededNodeTypes;
-}
-
 /**
  * Saves the static data if it changed
- *
  */
 export async function saveStaticData(workflow: Workflow): Promise<void> {
 	if (workflow.staticData.__dataChanged === true) {
@@ -402,12 +231,13 @@ export async function saveStaticData(workflow: Workflow): Promise<void> {
 		if (isWorkflowIdValid(workflow.id)) {
 			// Workflow is saved so update in database
 			try {
-				// eslint-disable-next-line @typescript-eslint/no-use-before-define
+				// eslint-disable-next-line @typescript-eslint/no-use-before-define, @typescript-eslint/no-non-null-assertion
 				await saveStaticDataById(workflow.id!, workflow.staticData);
 				workflow.staticData.__dataChanged = false;
 			} catch (error) {
 				ErrorReporter.error(error);
 				Logger.error(
+					// eslint-disable-next-line @typescript-eslint/restrict-template-expressions, @typescript-eslint/no-unsafe-member-access
 					`There was a problem saving the workflow with id "${workflow.id}" to save changed staticData: "${error.message}"`,
 					{ workflowId: workflow.id },
 				);
@@ -452,7 +282,6 @@ export async function getStaticDataById(workflowId: string | number) {
 
 /**
  * Set node ids if not already set
- *
  */
 export function addNodeIds(workflow: WorkflowEntity) {
 	const { nodes } = workflow;

packages/cli/src/WorkflowRunner.ts

@@ -26,22 +26,17 @@ import {
 	WorkflowOperationError,
 } from 'n8n-workflow';
 
-// eslint-disable-next-line import/no-extraneous-dependencies
 import PCancelable from 'p-cancelable';
 import { join as pathJoin } from 'path';
 import { fork } from 'child_process';
 
 import * as ActiveExecutions from '@/ActiveExecutions';
 import config from '@/config';
-import { CredentialsOverwrites } from '@/CredentialsOverwrites';
 import * as Db from '@/Db';
 import { ExternalHooks } from '@/ExternalHooks';
 import {
-	ICredentialsOverwrite,
-	ICredentialsTypeData,
 	IExecutionFlattedDb,
 	IProcessMessageDataHook,
-	ITransferNodeTypes,
 	IWorkflowExecutionDataProcess,
 	IWorkflowExecutionDataProcessWithExecution,
 } from '@/Interfaces';
@@ -60,8 +55,6 @@ import { PermissionChecker } from '@/UserManagement/PermissionChecker';
 export class WorkflowRunner {
 	activeExecutions: ActiveExecutions.ActiveExecutions;
 
-	credentialsOverwrites: ICredentialsOverwrite;
-
 	push: Push.Push;
 
 	jobQueue: Queue.JobQueue;
@@ -69,7 +62,6 @@ export class WorkflowRunner {
 	constructor() {
 		this.push = Push.getInstance();
 		this.activeExecutions = ActiveExecutions.getInstance();
-		this.credentialsOverwrites = CredentialsOverwrites().getAll();
 
 		const executionsMode = config.getEnv('executions.mode');
 
@@ -618,43 +610,7 @@ export class WorkflowRunner {
 		// Register the active execution
 		const executionId = await this.activeExecutions.add(data, subprocess, restartExecutionId);
 
-		// Check if workflow contains a "executeWorkflow" Node as in this
-		// case we can not know which nodeTypes and credentialTypes will
-		// be needed and so have to load all of them in the workflowRunnerProcess
-		let loadAllNodeTypes = false;
-		for (const node of data.workflowData.nodes) {
-			if (node.type === 'n8n-nodes-base.executeWorkflow' && node.disabled !== true) {
-				loadAllNodeTypes = true;
-				break;
-			}
-		}
-		let nodeTypeData: ITransferNodeTypes;
-		let credentialTypeData: ICredentialsTypeData;
-		// eslint-disable-next-line prefer-destructuring
-		let credentialsOverwrites = this.credentialsOverwrites;
-		if (loadAllNodeTypes) {
-			// Supply all nodeTypes and credentialTypes
-			nodeTypeData = WorkflowHelpers.getAllNodeTypeData();
-			credentialTypeData = WorkflowHelpers.getAllCredentalsTypeData();
-		} else {
-			// Supply only nodeTypes, credentialTypes and overwrites that the workflow needs
-			nodeTypeData = WorkflowHelpers.getNodeTypeData(data.workflowData.nodes);
-			credentialTypeData = WorkflowHelpers.getCredentialsDataByNodes(data.workflowData.nodes);
-
-			credentialsOverwrites = {};
-			for (const credentialName of Object.keys(credentialTypeData)) {
-				if (this.credentialsOverwrites[credentialName] !== undefined) {
-					credentialsOverwrites[credentialName] = this.credentialsOverwrites[credentialName];
-				}
-			}
-		}
-
 		(data as unknown as IWorkflowExecutionDataProcessWithExecution).executionId = executionId;
-		(data as unknown as IWorkflowExecutionDataProcessWithExecution).nodeTypeData = nodeTypeData;
-		(data as unknown as IWorkflowExecutionDataProcessWithExecution).credentialsOverwrite =
-			this.credentialsOverwrites;
-		(data as unknown as IWorkflowExecutionDataProcessWithExecution).credentialsTypeData =
-			credentialTypeData;
 
 		const workflowHooks = WorkflowExecuteAdditionalData.getWorkflowHooksMain(data, executionId);
 

packages/cli/src/WorkflowRunnerProcess.ts

@@ -11,15 +11,11 @@ import { BinaryDataManager, IProcessMessage, UserSettings, WorkflowExecute } fro
 import {
 	ErrorReporterProxy as ErrorReporter,
 	ExecutionError,
-	ICredentialType,
-	ICredentialTypeData,
 	IDataObject,
 	IExecuteResponsePromiseData,
 	IExecuteWorkflowInfo,
 	ILogger,
 	INodeExecutionData,
-	INodeType,
-	INodeTypeData,
 	IRun,
 	ITaskData,
 	IWorkflowExecuteAdditionalData,
@@ -39,6 +35,7 @@ import { ExternalHooks } from '@/ExternalHooks';
 import * as GenericHelpers from '@/GenericHelpers';
 import { IWorkflowExecuteProcess, IWorkflowExecutionDataProcessWithExecution } from '@/Interfaces';
 import { NodeTypes } from '@/NodeTypes';
+import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
 import * as WebhookHelpers from '@/WebhookHelpers';
 import * as WorkflowHelpers from '@/WorkflowHelpers';
 import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData';
@@ -46,12 +43,11 @@ import { getLogger } from '@/Logger';
 
 import config from '@/config';
 import { InternalHooksManager } from '@/InternalHooksManager';
-import { loadClassInIsolation } from '@/CommunityNodes/helpers';
 import { generateFailedExecutionFromError } from '@/WorkflowHelpers';
 import { initErrorHandling } from '@/ErrorReporting';
 import { PermissionChecker } from '@/UserManagement/PermissionChecker';
 
-export class WorkflowRunnerProcess {
+class WorkflowRunnerProcess {
 	data: IWorkflowExecutionDataProcessWithExecution | undefined;
 
 	logger: ILogger;
@@ -99,54 +95,15 @@ export class WorkflowRunnerProcess {
 
 		this.startedAt = new Date();
 
-		// Load the required nodes
-		const nodeTypesData: INodeTypeData = {};
-		// eslint-disable-next-line no-restricted-syntax
-		for (const nodeTypeName of Object.keys(this.data.nodeTypeData)) {
-			let tempNode: INodeType;
-			const { className, sourcePath } = this.data.nodeTypeData[nodeTypeName];
+		const loadNodesAndCredentials = LoadNodesAndCredentials();
+		await loadNodesAndCredentials.init();
 
-			try {
-				tempNode = loadClassInIsolation(sourcePath, className);
-			} catch (error) {
-				throw new Error(`Error loading node "${nodeTypeName}" from: "${sourcePath}"`);
-			}
-
-			nodeTypesData[nodeTypeName] = {
-				type: tempNode,
-				sourcePath,
-			};
-		}
-
-		const nodeTypes = NodeTypes();
-		await nodeTypes.init(nodeTypesData);
-
-		// Load the required credentials
-		const credentialsTypeData: ICredentialTypeData = {};
-		// eslint-disable-next-line no-restricted-syntax
-		for (const credentialTypeName of Object.keys(this.data.credentialsTypeData)) {
-			let tempCredential: ICredentialType;
-			const { className, sourcePath } = this.data.credentialsTypeData[credentialTypeName];
-
-			try {
-				tempCredential = loadClassInIsolation(sourcePath, className);
-			} catch (error) {
-				throw new Error(`Error loading credential "${credentialTypeName}" from: "${sourcePath}"`);
-			}
-
-			credentialsTypeData[credentialTypeName] = {
-				type: tempCredential,
-				sourcePath,
-			};
-		}
-
-		// Init credential types the workflow uses (is needed to apply default values to credentials)
-		const credentialTypes = CredentialTypes();
-		await credentialTypes.init(credentialsTypeData);
+		const nodeTypes = NodeTypes(loadNodesAndCredentials);
+		const credentialTypes = CredentialTypes(loadNodesAndCredentials);
 
 		// Load the credentials overwrites if any exist
-		const credentialsOverwrites = CredentialsOverwrites();
-		await credentialsOverwrites.init(inputData.credentialsOverwrite);
+		const credentialsOverwrites = CredentialsOverwrites(credentialTypes);
+		await credentialsOverwrites.init();
 
 		// Load all external hooks
 		const externalHooks = ExternalHooks();

packages/cli/src/api/executions.api.ts

@@ -1,5 +1,4 @@
 /* eslint-disable no-restricted-syntax */
-/* eslint-disable import/no-extraneous-dependencies */
 /* eslint-disable @typescript-eslint/restrict-template-expressions */
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
@@ -9,9 +8,9 @@
 /* eslint-disable @typescript-eslint/no-unused-vars */
 import express from 'express';
 import { validate as jsonSchemaValidate } from 'jsonschema';
-import _, { cloneDeep } from 'lodash';
 import { BinaryDataManager } from 'n8n-core';
 import {
+	deepCopy,
 	IDataObject,
 	IWorkflowBase,
 	JsonObject,
@@ -259,7 +258,7 @@ executionsController.get(
 			query = query.andWhere(filter);
 		}
 
-		const countFilter = cloneDeep(filter ?? {});
+		const countFilter = deepCopy(filter ?? {});
 		countFilter.id = Not(In(executingWorkflowIds));
 
 		const executions = await query.getMany();

packages/cli/src/api/nodeTypes.api.ts

@@ -1,15 +1,8 @@
-/* eslint-disable import/no-extraneous-dependencies */
 import express from 'express';
 import { readFile } from 'fs/promises';
-import _ from 'lodash';
+import get from 'lodash.get';
 
-import {
-	ICredentialType,
-	INodeType,
-	INodeTypeDescription,
-	INodeTypeNameVersion,
-	NodeHelpers,
-} from 'n8n-workflow';
+import type { ICredentialType, INodeTypeDescription, INodeTypeNameVersion } from 'n8n-workflow';
 
 import { CredentialTypes } from '@/CredentialTypes';
 import config from '@/config';
@@ -74,50 +67,11 @@ function injectCustomApiCallOption(description: INodeTypeDescription) {
 
 export const nodeTypesController = express.Router();
 
-// Returns all the node-types
-nodeTypesController.get(
-	'/',
-	ResponseHelper.send(async (req: express.Request): Promise<INodeTypeDescription[]> => {
-		const returnData: INodeTypeDescription[] = [];
-		const onlyLatest = req.query.onlyLatest === 'true';
-
-		const nodeTypes = NodeTypes();
-		const allNodes = nodeTypes.getAll();
-
-		const getNodeDescription = (nodeType: INodeType): INodeTypeDescription => {
-			const nodeInfo: INodeTypeDescription = { ...nodeType.description };
-			if (req.query.includeProperties !== 'true') {
-				// @ts-ignore
-				delete nodeInfo.properties;
-			}
-			return nodeInfo;
-		};
-
-		if (onlyLatest) {
-			allNodes.forEach((nodeData) => {
-				const nodeType = NodeHelpers.getVersionedNodeType(nodeData);
-				const nodeInfo: INodeTypeDescription = getNodeDescription(nodeType);
-				returnData.push(nodeInfo);
-			});
-		} else {
-			allNodes.forEach((nodeData) => {
-				const allNodeTypes = NodeHelpers.getVersionedNodeTypeAll(nodeData);
-				allNodeTypes.forEach((element) => {
-					const nodeInfo: INodeTypeDescription = getNodeDescription(element);
-					returnData.push(nodeInfo);
-				});
-			});
-		}
-
-		return returnData;
-	}),
-);
-
 // Returns node information based on node names and versions
 nodeTypesController.post(
 	'/',
 	ResponseHelper.send(async (req: express.Request): Promise<INodeTypeDescription[]> => {
-		const nodeInfos = _.get(req, 'body.nodeInfos', []) as INodeTypeNameVersion[];
+		const nodeInfos = get(req, 'body.nodeInfos', []) as INodeTypeNameVersion[];
 
 		const defaultLocale = config.getEnv('defaultLocale');
 

packages/cli/src/commands/execute.ts

@@ -123,19 +123,19 @@ export class Execute extends Command {
 		// Wait till the n8n-packages have been read
 		await loadNodesAndCredentialsPromise;
 
+		NodeTypes(loadNodesAndCredentials);
+		const credentialTypes = CredentialTypes(loadNodesAndCredentials);
+
 		// Load the credentials overwrites if any exist
-		const credentialsOverwrites = CredentialsOverwrites();
-		await credentialsOverwrites.init();
+		await CredentialsOverwrites(credentialTypes).init();
 
 		// Load all external hooks
 		const externalHooks = ExternalHooks();
 		await externalHooks.init();
 
 		// Add the found types to an instance other parts of the application can use
-		const nodeTypes = NodeTypes();
-		await nodeTypes.init(loadNodesAndCredentials.nodeTypes);
-		const credentialTypes = CredentialTypes();
-		await credentialTypes.init(loadNodesAndCredentials.credentialTypes);
+		const nodeTypes = NodeTypes(loadNodesAndCredentials);
+		CredentialTypes(loadNodesAndCredentials);
 
 		const instanceId = await UserSettings.getInstanceId();
 		const { cli } = await GenericHelpers.getVersions();

packages/cli/src/commands/executeBatch.ts

@@ -17,8 +17,7 @@ import { sep } from 'path';
 
 import { diff } from 'json-diff';
 
-// eslint-disable-next-line import/no-extraneous-dependencies
-import { pick } from 'lodash';
+import pick from 'lodash.pick';
 import { getLogger } from '@/Logger';
 
 import * as ActiveExecutions from '@/ActiveExecutions';
@@ -312,18 +311,19 @@ export class ExecuteBatch extends Command {
 		// Wait till the n8n-packages have been read
 		await loadNodesAndCredentialsPromise;
 
+		NodeTypes(loadNodesAndCredentials);
+		const credentialTypes = CredentialTypes(loadNodesAndCredentials);
+
 		// Load the credentials overwrites if any exist
-		await CredentialsOverwrites().init();
+		await CredentialsOverwrites(credentialTypes).init();
 
 		// Load all external hooks
 		const externalHooks = ExternalHooks();
 		await externalHooks.init();
 
 		// Add the found types to an instance other parts of the application can use
-		const nodeTypes = NodeTypes();
-		await nodeTypes.init(loadNodesAndCredentials.nodeTypes);
-		const credentialTypes = CredentialTypes();
-		await credentialTypes.init(loadNodesAndCredentials.credentialTypes);
+		const nodeTypes = NodeTypes(loadNodesAndCredentials);
+		CredentialTypes(loadNodesAndCredentials);
 
 		const instanceId = await UserSettings.getInstanceId();
 		const { cli } = await GenericHelpers.getVersions();

packages/cli/src/commands/start.ts

@@ -9,7 +9,6 @@
 import localtunnel from 'localtunnel';
 import { BinaryDataManager, TUNNEL_SUBDOMAIN_ENV, UserSettings } from 'n8n-core';
 import { Command, flags } from '@oclif/command';
-// eslint-disable-next-line import/no-extraneous-dependencies
 import Redis from 'ioredis';
 
 import { IDataObject, LoggerProxy, sleep } from 'n8n-workflow';
@@ -115,7 +114,7 @@ export class Start extends Command {
 			await InternalHooksManager.getInstance().onN8nStop();
 
 			const skipWebhookDeregistration = config.getEnv(
-				'endpoints.skipWebhoooksDeregistrationOnShutdown',
+				'endpoints.skipWebhooksDeregistrationOnShutdown',
 			);
 
 			const removePromises = [];
@@ -210,14 +209,13 @@ export class Start extends Command {
 				await externalHooks.init();
 
 				// Add the found types to an instance other parts of the application can use
-				const nodeTypes = NodeTypes();
-				await nodeTypes.init(loadNodesAndCredentials.nodeTypes);
-				const credentialTypes = CredentialTypes();
-				await credentialTypes.init(loadNodesAndCredentials.credentialTypes);
+				const nodeTypes = NodeTypes(loadNodesAndCredentials);
+				const credentialTypes = CredentialTypes(loadNodesAndCredentials);
 
 				// Load the credentials overwrites if any exist
-				const credentialsOverwrites = CredentialsOverwrites();
-				await credentialsOverwrites.init();
+				await CredentialsOverwrites(credentialTypes).init();
+
+				await loadNodesAndCredentials.generateTypesForFrontend();
 
 				// Wait till the database is ready
 				await startDbInitPromise;
@@ -227,13 +225,13 @@ export class Start extends Command {
 					packageName: string;
 					version: string;
 				}>();
-				installedPackages.forEach((installedpackage) => {
-					installedpackage.installedNodes.forEach((installedNode) => {
-						if (!loadNodesAndCredentials.nodeTypes[installedNode.type]) {
+				installedPackages.forEach((installedPackage) => {
+					installedPackage.installedNodes.forEach((installedNode) => {
+						if (!loadNodesAndCredentials.known.nodes[installedNode.type]) {
 							// Leave the list ready for installing in case we need.
 							missingPackages.add({
-								packageName: installedpackage.packageName,
-								version: installedpackage.installedVersion,
+								packageName: installedPackage.packageName,
+								version: installedPackage.installedVersion,
 							});
 						}
 					});

packages/cli/src/commands/webhook.ts

@@ -6,7 +6,6 @@
 /* eslint-disable @typescript-eslint/unbound-method */
 import { BinaryDataManager, UserSettings } from 'n8n-core';
 import { Command, flags } from '@oclif/command';
-// eslint-disable-next-line import/no-extraneous-dependencies
 import Redis from 'ioredis';
 
 import { IDataObject, LoggerProxy, sleep } from 'n8n-workflow';
@@ -132,26 +131,23 @@ export class Webhook extends Command {
 
 				// Make sure the settings exist
 				// eslint-disable-next-line @typescript-eslint/no-unused-vars
-				const userSettings = await UserSettings.prepareUserSettings();
+				await UserSettings.prepareUserSettings();
 
 				// Load all node and credential types
 				const loadNodesAndCredentials = LoadNodesAndCredentials();
 				await loadNodesAndCredentials.init();
 
+				// Add the found types to an instance other parts of the application can use
+				const nodeTypes = NodeTypes(loadNodesAndCredentials);
+				const credentialTypes = CredentialTypes(loadNodesAndCredentials);
+
 				// Load the credentials overwrites if any exist
-				const credentialsOverwrites = CredentialsOverwrites();
-				await credentialsOverwrites.init();
+				await CredentialsOverwrites(credentialTypes).init();
 
 				// Load all external hooks
 				const externalHooks = ExternalHooks();
 				await externalHooks.init();
 
-				// Add the found types to an instance other parts of the application can use
-				const nodeTypes = NodeTypes();
-				await nodeTypes.init(loadNodesAndCredentials.nodeTypes);
-				const credentialTypes = CredentialTypes();
-				await credentialTypes.init(loadNodesAndCredentials.credentialTypes);
-
 				// Wait till the database is ready
 				await startDbInitPromise;
 

packages/cli/src/commands/worker.ts

@@ -7,7 +7,6 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 /* eslint-disable @typescript-eslint/restrict-template-expressions */
 /* eslint-disable @typescript-eslint/no-unused-vars */
-// eslint-disable-next-line import/no-extraneous-dependencies
 import express from 'express';
 import http from 'http';
 import PCancelable from 'p-cancelable';
@@ -283,20 +282,17 @@ export class Worker extends Command {
 				const loadNodesAndCredentials = LoadNodesAndCredentials();
 				await loadNodesAndCredentials.init();
 
+				// Add the found types to an instance other parts of the application can use
+				const nodeTypes = NodeTypes(loadNodesAndCredentials);
+				const credentialTypes = CredentialTypes(loadNodesAndCredentials);
+
 				// Load the credentials overwrites if any exist
-				const credentialsOverwrites = CredentialsOverwrites();
-				await credentialsOverwrites.init();
+				await CredentialsOverwrites(credentialTypes).init();
 
 				// Load all external hooks
 				const externalHooks = ExternalHooks();
 				await externalHooks.init();
 
-				// Add the found types to an instance other parts of the application can use
-				const nodeTypes = NodeTypes();
-				await nodeTypes.init(loadNodesAndCredentials.nodeTypes);
-				const credentialTypes = CredentialTypes();
-				await credentialTypes.init(loadNodesAndCredentials.credentialTypes);
-
 				// Wait till the database is ready
 				await startDbInitPromise;
 

packages/cli/src/config/schema.ts

@@ -580,9 +580,9 @@ export const schema = {
 			env: 'N8N_DISABLE_PRODUCTION_MAIN_PROCESS',
 			doc: 'Disable production webhooks from main process. This helps ensures no http traffic load to main process when using webhook-specific processes.',
 		},
-		skipWebhoooksDeregistrationOnShutdown: {
+		skipWebhooksDeregistrationOnShutdown: {
 			/**
-			 * Longer explanation: n8n deregisters webhooks on shutdown / deactivation
+			 * Longer explanation: n8n de-registers webhooks on shutdown / deactivation
 			 * and registers on startup / activation. If we skip
 			 * deactivation on shutdown, webhooks will remain active on 3rd party services.
 			 * We don't have to worry about startup as it always

packages/cli/src/constants.ts

@@ -2,11 +2,12 @@
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 /* eslint-disable @typescript-eslint/naming-convention */
 import { resolve, join } from 'path';
-import { RESPONSE_ERROR_MESSAGES as CORE_RESPONSE_ERROR_MESSAGES } from 'n8n-core';
+import { RESPONSE_ERROR_MESSAGES as CORE_RESPONSE_ERROR_MESSAGES, UserSettings } from 'n8n-core';
 
 export const CLI_DIR = resolve(__dirname, '..');
 export const TEMPLATES_DIR = join(CLI_DIR, 'templates');
 export const NODES_BASE_DIR = join(CLI_DIR, '..', 'nodes-base');
+export const GENERATED_STATIC_DIR = join(UserSettings.getUserHome(), '.cache/n8n/public');
 
 export const NODE_PACKAGE_PREFIX = 'n8n-nodes-';
 
@@ -14,6 +15,7 @@ export const STARTER_TEMPLATE_NAME = `${NODE_PACKAGE_PREFIX}starter`;
 
 export const RESPONSE_ERROR_MESSAGES = {
 	NO_CREDENTIAL: 'Credential not found',
+	NO_NODE: 'Node not found',
 	NO_ENCRYPTION_KEY: CORE_RESPONSE_ERROR_MESSAGES.NO_ENCRYPTION_KEY,
 	PACKAGE_NAME_NOT_PROVIDED: 'Package name is required',
 	PACKAGE_NAME_NOT_VALID: `Package name is not valid - it must start with "${NODE_PACKAGE_PREFIX}"`,

packages/cli/src/databases/entities/Settings.ts

@@ -1,7 +1,7 @@
 import type { IDataObject } from 'n8n-workflow';
 import { Column, Entity, PrimaryColumn } from 'typeorm';
 
-export interface ISettingsDb {
+interface ISettingsDb {
 	key: string;
 	value: string | boolean | IDataObject | number;
 	loadOnStartup: boolean;

packages/cli/src/databases/utils/migrationHelpers.ts

@@ -65,7 +65,7 @@ export function logMigrationEnd(
 	}, 100);
 }
 
-export function batchQuery(query: string, limit: number, offset = 0): string {
+function batchQuery(query: string, limit: number, offset = 0): string {
 	return `
 			${query}
 			LIMIT ${limit}

packages/cli/src/requests.d.ts

@@ -10,7 +10,7 @@ import {
 	IWorkflowSettings,
 } from 'n8n-workflow';
 
-import type { IExecutionDeleteFilter, IWorkflowDb } from '.';
+import type { IExecutionDeleteFilter, IWorkflowDb } from '@/Interfaces';
 import type { Role } from '@db/entities/Role';
 import type { User } from '@db/entities/User';
 import * as UserManagementMailer from '@/UserManagement/email/UserManagementMailer';