mohiit1502/Automata
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(core): Refactor nodes loading (no-changelog) (#7283)

Browse Source 
fixes PAY-605
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™
2023-10-09 16:09:23 +02:00
committed by GitHub
parent 789e1e7ed4
commit c5ee06cc61
31 changed files with 603 additions and 683 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
packages/cli/src/CredentialTypes.ts
View File

@@ -1,17 +1,16 @@
import { Service } from 'typedi';
import { loadClassInIsolation } from 'n8n-core';
import type { ICredentialType, ICredentialTypes, LoadedClass } from 'n8n-workflow';
import { Service } from 'typedi';
import { RESPONSE_ERROR_MESSAGES } from './constants';
import { LoadNodesAndCredentials } from './LoadNodesAndCredentials';
import { RESPONSE_ERROR_MESSAGES } from '@/constants';
import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
@Service()
export class CredentialTypes implements ICredentialTypes {
constructor(private nodesAndCredentials: LoadNodesAndCredentials) {
nodesAndCredentials.credentialTypes = this;
}
constructor(private loadNodesAndCredentials: LoadNodesAndCredentials) {}
recognizes(type: string) {
return type in this.knownCredentials || type in this.loadedCredentials;
const { loadedCredentials, knownCredentials } = this.loadNodesAndCredentials;
return type in knownCredentials || type in loadedCredentials;
}
getByName(credentialType: string): ICredentialType {
@@ -19,14 +18,14 @@ export class CredentialTypes implements ICredentialTypes {
}
getNodeTypesToTestWith(type: string): string[] {
return this.knownCredentials[type]?.nodesToTestWith ?? [];
return this.loadNodesAndCredentials.knownCredentials[type]?.nodesToTestWith ?? [];
}
/**
* Returns all parent types of the given credential type
*/
getParentTypes(typeName: string): string[] {
const extendsArr = this.knownCredentials[typeName]?.extends ?? [];
const extendsArr = this.loadNodesAndCredentials.knownCredentials[typeName]?.extends ?? [];
if (extendsArr.length) {
extendsArr.forEach((type) => {
extendsArr.push(...this.getParentTypes(type));
@@ -36,12 +35,11 @@ export class CredentialTypes implements ICredentialTypes {
}
private getCredential(type: string): LoadedClass<ICredentialType> {
const loadedCredentials = this.loadedCredentials;
const { loadedCredentials, knownCredentials } = this.loadNodesAndCredentials;
if (type in loadedCredentials) {
return loadedCredentials[type];
}
const knownCredentials = this.knownCredentials;
if (type in knownCredentials) {
const { className, sourcePath } = knownCredentials[type];
const loaded: ICredentialType = loadClassInIsolation(sourcePath, className);
@@ -50,12 +48,4 @@ export class CredentialTypes implements ICredentialTypes {
}
throw new Error(`${RESPONSE_ERROR_MESSAGES.NO_CREDENTIAL}: ${type}`);
}
private get loadedCredentials() {
return this.nodesAndCredentials.loaded.credentials;
}
private get knownCredentials() {
return this.nodesAndCredentials.known.credentials;
}
}

18
packages/cli/src/CredentialsHelper.ts
View File

@@ -1,5 +1,4 @@
/* eslint-disable @typescript-eslint/no-unsafe-argument */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/no-unsafe-return */
@@ -89,13 +88,11 @@ const mockNodeTypes: INodeTypes = {
};
export class CredentialsHelper extends ICredentialsHelper {
constructor(
encryptionKey: string,
private credentialTypes = Container.get(CredentialTypes),
private nodeTypes = Container.get(NodeTypes),
) {
super(encryptionKey);
}
private credentialTypes = Container.get(CredentialTypes);
private nodeTypes = Container.get(NodeTypes);
private credentialsOverwrites = Container.get(CredentialsOverwrites);
/**
* Add the required authentication information to the request
@@ -388,7 +385,10 @@ export class CredentialsHelper extends ICredentialsHelper {
const credentialsProperties = this.getCredentialsProperties(type);
// Load and apply the credentials overwrites if any exist
const dataWithOverwrites = CredentialsOverwrites().applyOverwrite(type, decryptedDataOriginal);
const dataWithOverwrites = this.credentialsOverwrites.applyOverwrite(
type,
decryptedDataOriginal,
);
// Add the default credential values
let decryptedData = NodeHelpers.getNodeParameters(

28
packages/cli/src/CredentialsOverwrites.ts
View File

@@ -1,14 +1,17 @@
import config from '@/config';
import { Service } from 'typedi';
import type { ICredentialDataDecryptedObject } from 'n8n-workflow';
import { deepCopy, LoggerProxy as Logger, jsonParse, ICredentialTypes } from 'n8n-workflow';
import { deepCopy, LoggerProxy as Logger, jsonParse } from 'n8n-workflow';
import config from '@/config';
import type { ICredentialsOverwrite } from '@/Interfaces';
import { CredentialTypes } from '@/CredentialTypes';
class CredentialsOverwritesClass {
@Service()
export class CredentialsOverwrites {
private overwriteData: ICredentialsOverwrite = {};
private resolvedTypes: string[] = [];
constructor(private credentialTypes: ICredentialTypes) {
constructor(private credentialTypes: CredentialTypes) {
const data = config.getEnv('credentials.overwrite.data');
const overwriteData = jsonParse<ICredentialsOverwrite>(data, {
errorMessage: 'The credentials-overwrite is not valid JSON.',
@@ -96,20 +99,3 @@ class CredentialsOverwritesClass {
return this.overwriteData;
}
}
let credentialsOverwritesInstance: CredentialsOverwritesClass | undefined;
// eslint-disable-next-line @typescript-eslint/naming-convention
export function CredentialsOverwrites(
credentialTypes?: ICredentialTypes,
): CredentialsOverwritesClass {
if (!credentialsOverwritesInstance) {
if (credentialTypes) {
credentialsOverwritesInstance = new CredentialsOverwritesClass(credentialTypes);
} else {
throw new Error('CredentialsOverwrites not initialized yet');
}
}
return credentialsOverwritesInstance;
}

230
packages/cli/src/LoadNodesAndCredentials.ts
View File

@@ -1,5 +1,8 @@
import uniq from 'lodash/uniq';
import glob from 'fast-glob';
import { Container, Service } from 'typedi';
import path from 'path';
import fsPromises from 'fs/promises';
import type { DirectoryLoader, Types } from 'n8n-core';
import {
CUSTOM_EXTENSION_ENV,
@@ -9,36 +12,30 @@ import {
LazyPackageDirectoryLoader,
} from 'n8n-core';
import type {
ICredentialTypes,
ILogger,
INodesAndCredentials,
KnownNodesAndCredentials,
INodeTypeDescription,
LoadedNodesAndCredentials,
INodeTypeData,
ICredentialTypeData,
} from 'n8n-workflow';
import { LoggerProxy, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow';
import { createWriteStream } from 'fs';
import { mkdir } from 'fs/promises';
import path from 'path';
import config from '@/config';
import type { InstalledPackages } from '@db/entities/InstalledPackages';
import { CommunityPackageService } from './services/communityPackage.service';
import {
GENERATED_STATIC_DIR,
RESPONSE_ERROR_MESSAGES,
CUSTOM_API_CALL_KEY,
CUSTOM_API_CALL_NAME,
inTest,
CLI_DIR,
inE2ETests,
} from '@/constants';
import { CredentialsOverwrites } from '@/CredentialsOverwrites';
import Container, { Service } from 'typedi';
interface LoadedNodesAndCredentials {
nodes: INodeTypeData;
credentials: ICredentialTypeData;
}
@Service()
export class LoadNodesAndCredentials implements INodesAndCredentials {
known: KnownNodesAndCredentials = { nodes: {}, credentials: {} };
export class LoadNodesAndCredentials {
private known: KnownNodesAndCredentials = { nodes: {}, credentials: {} };
loaded: LoadedNodesAndCredentials = { nodes: {}, credentials: {} };
@@ -50,20 +47,20 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
includeNodes = config.getEnv('nodes.include');
credentialTypes: ICredentialTypes;
logger: ILogger;
private downloadFolder: string;
private postProcessors: Array<() => Promise<void>> = [];
async init() {
if (inTest) throw new Error('Not available in tests');
// Make sure the imported modules can resolve dependencies fine.
const delimiter = process.platform === 'win32' ? ';' : ':';
process.env.NODE_PATH = module.paths.join(delimiter);
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
if (!inTest) module.constructor._initPaths();
module.constructor._initPaths();
if (!inE2ETests) {
this.excludeNodes = this.excludeNodes ?? [];
@@ -91,48 +88,30 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
await this.loadNodesFromCustomDirectories();
await this.postProcessLoaders();
this.injectCustomApiCallOptions();
}
async generateTypesForFrontend() {
const credentialsOverwrites = CredentialsOverwrites().getAll();
for (const credential of this.types.credentials) {
const overwrittenProperties = [];
this.credentialTypes
.getParentTypes(credential.name)
.reverse()
.map((name) => credentialsOverwrites[name])
.forEach((overwrite) => {
if (overwrite) overwrittenProperties.push(...Object.keys(overwrite));
});
addPostProcessor(fn: () => Promise<void>) {
this.postProcessors.push(fn);
}
if (credential.name in credentialsOverwrites) {
overwrittenProperties.push(...Object.keys(credentialsOverwrites[credential.name]));
}
isKnownNode(type: string) {
return type in this.known.nodes;
}
if (overwrittenProperties.length) {
credential.__overwrittenProperties = uniq(overwrittenProperties);
}
}
get loadedCredentials() {
return this.loaded.credentials;
}
// pre-render all the node and credential types as static json files
await mkdir(path.join(GENERATED_STATIC_DIR, 'types'), { recursive: true });
get loadedNodes() {
return this.loaded.nodes;
}
const writeStaticJSON = async (name: string, data: object[]) => {
const filePath = path.join(GENERATED_STATIC_DIR, `types/${name}.json`);
const stream = createWriteStream(filePath, 'utf-8');
stream.write('[\n');
data.forEach((entry, index) => {
stream.write(JSON.stringify(entry));
if (index !== data.length - 1) stream.write(',');
stream.write('\n');
});
stream.write(']\n');
stream.end();
};
get knownCredentials() {
return this.known.credentials;
}
await writeStaticJSON('nodes', this.types.nodes);
await writeStaticJSON('credentials', this.types.credentials);
get knownNodes() {
return this.known.nodes;
}
private async loadNodesFromNodeModules(
@@ -163,6 +142,18 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
}
}
resolveIcon(packageName: string, url: string): string | undefined {
const loader = this.loaders[packageName];
if (loader) {
const pathPrefix = `/icons/${packageName}/`;
const filePath = path.resolve(loader.directory, url.substring(pathPrefix.length));
if (!path.relative(loader.directory, filePath).includes('..')) {
return filePath;
}
}
return undefined;
}
getCustomDirectories(): string[] {
const customDirectories = [UserSettings.getUserN8nFolderCustomExtensionPath()];
@@ -180,93 +171,16 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
}
}
private async installOrUpdateNpmModule(
packageName: string,
options: { version?: string } | { installedPackage: InstalledPackages },
) {
const isUpdate = 'installedPackage' in options;
const command = isUpdate
? `npm update ${packageName}`
: `npm install ${packageName}${options.version ? `@${options.version}` : ''}`;
const communityPackageService = Container.get(CommunityPackageService);
try {
await communityPackageService.executeNpmCommand(command);
} catch (error) {
if (error instanceof Error && error.message === RESPONSE_ERROR_MESSAGES.PACKAGE_NOT_FOUND) {
throw new Error(`The npm package "${packageName}" could not be found.`);
}
throw error;
}
async loadPackage(packageName: string) {
const finalNodeUnpackedPath = path.join(this.downloadFolder, 'node_modules', packageName);
let loader: PackageDirectoryLoader;
try {
loader = await this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);
} catch (error) {
// Remove this package since loading it failed
const removeCommand = `npm remove ${packageName}`;
try {
await communityPackageService.executeNpmCommand(removeCommand);
} catch {}
throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_LOADING_FAILED, { cause: error });
}
if (loader.loadedNodes.length > 0) {
// Save info to DB
try {
if (isUpdate) {
await communityPackageService.removePackageFromDatabase(options.installedPackage);
}
const installedPackage = await communityPackageService.persistInstalledPackage(loader);
await this.postProcessLoaders();
await this.generateTypesForFrontend();
return installedPackage;
} catch (error) {
LoggerProxy.error('Failed to save installed packages and nodes', {
error: error as Error,
packageName,
});
throw error;
}
} else {
// Remove this package since it contains no loadable nodes
const removeCommand = `npm remove ${packageName}`;
try {
await communityPackageService.executeNpmCommand(removeCommand);
} catch {}
throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_DOES_NOT_CONTAIN_NODES);
}
return this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);
}
async installNpmModule(packageName: string, version?: string): Promise<InstalledPackages> {
return this.installOrUpdateNpmModule(packageName, { version });
}
async removeNpmModule(packageName: string, installedPackage: InstalledPackages): Promise<void> {
const communityPackageService = Container.get(CommunityPackageService);
await communityPackageService.executeNpmCommand(`npm remove ${packageName}`);
await communityPackageService.removePackageFromDatabase(installedPackage);
async unloadPackage(packageName: string) {
if (packageName in this.loaders) {
this.loaders[packageName].reset();
delete this.loaders[packageName];
}
await this.postProcessLoaders();
await this.generateTypesForFrontend();
}
async updateNpmModule(
packageName: string,
installedPackage: InstalledPackages,
): Promise<InstalledPackages> {
return this.installOrUpdateNpmModule(packageName, { installedPackage });
}
/**
@@ -382,5 +296,49 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
};
}
}
this.injectCustomApiCallOptions();
for (const postProcessor of this.postProcessors) {
await postProcessor();
}
}
async setupHotReload() {
const { default: debounce } = await import('lodash/debounce');
// eslint-disable-next-line import/no-extraneous-dependencies
const { watch } = await import('chokidar');
// eslint-disable-next-line @typescript-eslint/naming-convention
const { Push } = await import('@/push');
const push = Container.get(Push);
Object.values(this.loaders).forEach(async (loader) => {
try {
await fsPromises.access(loader.directory);
} catch {
// If directory doesn't exist, there is nothing to watch
return;
}
const realModulePath = path.join(await fsPromises.realpath(loader.directory), path.sep);
const reloader = debounce(async () => {
const modulesToUnload = Object.keys(require.cache).filter((filePath) =>
filePath.startsWith(realModulePath),
);
modulesToUnload.forEach((filePath) => {
delete require.cache[filePath];
});
loader.reset();
await loader.loadAll();
await this.postProcessLoaders();
push.send('nodeDescriptionUpdated', undefined);
}, 100);
const toWatch = loader.isLazyLoaded
? ['**/nodes.json', '**/credentials.json']
: ['**/*.js', '**/*.json'];
watch(toWatch, { cwd: realModulePath }).on('change', reloader);
});
}
}

22
packages/cli/src/NodeTypes.ts
View File

@@ -16,12 +16,8 @@ import type { Dirent } from 'fs';
@Service()
export class NodeTypes implements INodeTypes {
constructor(private nodesAndCredentials: LoadNodesAndCredentials) {}
init() {
// Some nodeTypes need to get special parameters applied like the
// polling nodes the polling times
this.applySpecialNodeParameters();
constructor(private loadNodesAndCredentials: LoadNodesAndCredentials) {
loadNodesAndCredentials.addPostProcessor(async () => this.applySpecialNodeParameters());
}
/**
@@ -50,20 +46,20 @@ export class NodeTypes implements INodeTypes {
return NodeHelpers.getVersionedNodeType(this.getNode(nodeType).type, version);
}
/* Some nodeTypes need to get special parameters applied like the polling nodes the polling times */
applySpecialNodeParameters() {
for (const nodeTypeData of Object.values(this.loadedNodes)) {
for (const nodeTypeData of Object.values(this.loadNodesAndCredentials.loadedNodes)) {
const nodeType = NodeHelpers.getVersionedNodeType(nodeTypeData.type);
NodeHelpers.applySpecialNodeParameters(nodeType);
}
}
private getNode(type: string): LoadedClass<INodeType | IVersionedNodeType> {
const loadedNodes = this.loadedNodes;
const { loadedNodes, knownNodes } = this.loadNodesAndCredentials;
if (type in loadedNodes) {
return loadedNodes[type];
}
const knownNodes = this.knownNodes;
if (type in knownNodes) {
const { className, sourcePath } = knownNodes[type];
const loaded: INodeType = loadClassInIsolation(sourcePath, className);
@@ -74,14 +70,6 @@ export class NodeTypes implements INodeTypes {
throw new Error(`${RESPONSE_ERROR_MESSAGES.NO_NODE}: ${type}`);
}
private get loadedNodes() {
return this.nodesAndCredentials.loaded.nodes;
}
private get knownNodes() {
return this.nodesAndCredentials.known.nodes;
}
async getNodeTranslationPath({
nodeSourcePath,
longNodeType,

47
packages/cli/src/ReloadNodesAndCredentials.ts
View File

@@ -1,47 +0,0 @@
import path from 'path';
import { realpath, access } from 'fs/promises';
import type { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
import type { NodeTypes } from '@/NodeTypes';
import type { Push } from '@/push';
export const reloadNodesAndCredentials = async (
loadNodesAndCredentials: LoadNodesAndCredentials,
nodeTypes: NodeTypes,
push: Push,
) => {
const { default: debounce } = await import('lodash/debounce');
// eslint-disable-next-line import/no-extraneous-dependencies
const { watch } = await import('chokidar');
Object.values(loadNodesAndCredentials.loaders).forEach(async (loader) => {
try {
await access(loader.directory);
} catch {
// If directory doesn't exist, there is nothing to watch
return;
}
const realModulePath = path.join(await realpath(loader.directory), path.sep);
const reloader = debounce(async () => {
const modulesToUnload = Object.keys(require.cache).filter((filePath) =>
filePath.startsWith(realModulePath),
);
modulesToUnload.forEach((filePath) => {
delete require.cache[filePath];
});
loader.reset();
await loader.loadAll();
await loadNodesAndCredentials.postProcessLoaders();
await loadNodesAndCredentials.generateTypesForFrontend();
nodeTypes.applySpecialNodeParameters();
push.send('nodeDescriptionUpdated', undefined);
}, 100);
const toWatch = loader.isLazyLoaded
? ['**/nodes.json', '**/credentials.json']
: ['**/*.js', '**/*.json'];
watch(toWatch, { cwd: realModulePath }).on('change', reloader);
});
};

51
packages/cli/src/Server.ts
View File

@@ -11,7 +11,7 @@ import assert from 'assert';
import { exec as callbackExec } from 'child_process';
import { access as fsAccess } from 'fs/promises';
import os from 'os';
import { join as pathJoin, resolve as pathResolve, relative as pathRelative } from 'path';
import { join as pathJoin, resolve as pathResolve } from 'path';
import { createHmac } from 'crypto';
import { promisify } from 'util';
import cookieParser from 'cookie-parser';
@@ -86,7 +86,6 @@ import {
LdapController,
MeController,
MFAController,
NodesController,
NodeTypesController,
OwnerController,
PasswordResetController,
@@ -172,6 +171,7 @@ import type { ExecutionEntity } from '@db/entities/ExecutionEntity';
import { TOTPService } from './Mfa/totp.service';
import { MfaService } from './Mfa/mfa.service';
import { handleMfaDisable, isMfaFeatureEnabled } from './Mfa/helpers';
import type { FrontendService } from './services/frontend.service';
import { JwtService } from './services/jwt.service';
import { RoleService } from './services/role.service';
import { UserService } from './services/user.service';
@@ -202,6 +202,8 @@ export class Server extends AbstractServer {
credentialTypes: ICredentialTypes;
frontendService: FrontendService;
postHog: PostHogClient;
push: Push;
@@ -362,6 +364,16 @@ export class Server extends AbstractServer {
this.credentialTypes = Container.get(CredentialTypes);
this.nodeTypes = Container.get(NodeTypes);
if (!config.getEnv('endpoints.disableUi')) {
// eslint-disable-next-line @typescript-eslint/naming-convention
const { FrontendService } = await import('@/services/frontend.service');
this.frontendService = Container.get(FrontendService);
this.loadNodesAndCredentials.addPostProcessor(async () =>
this.frontendService.generateTypes(),
);
await this.frontendService.generateTypes();
}
this.activeExecutionsInstance = Container.get(ActiveExecutions);
this.waitTracker = Container.get(WaitTracker);
this.postHog = Container.get(PostHogClient);
@@ -419,8 +431,7 @@ export class Server extends AbstractServer {
};
if (inDevelopment && process.env.N8N_DEV_RELOAD === 'true') {
const { reloadNodesAndCredentials } = await import('@/ReloadNodesAndCredentials');
await reloadNodesAndCredentials(this.loadNodesAndCredentials, this.nodeTypes, this.push);
void this.loadNodesAndCredentials.setupHotReload();
}
void Db.collections.Workflow.findOne({
@@ -435,7 +446,7 @@ export class Server extends AbstractServer {
/**
* Returns the current settings for the frontend
*/
getSettingsForFrontend(): IN8nUISettings {
private async getSettingsForFrontend(): Promise<IN8nUISettings> {
// Update all urls, in case `WEBHOOK_URL` was updated by `--tunnel`
const instanceBaseUrl = getInstanceBaseUrl();
this.frontendSettings.urlBaseWebhook = WebhookHelpers.getWebhookBaseUrl();
@@ -506,8 +517,11 @@ export class Server extends AbstractServer {
});
}
if (config.get('nodes.packagesMissing').length > 0) {
this.frontendSettings.missingPackages = true;
if (config.getEnv('nodes.communityPackages.enabled')) {
// eslint-disable-next-line @typescript-eslint/naming-convention
const { CommunityPackagesService } = await import('@/services/communityPackages.service');
this.frontendSettings.missingPackages =
Container.get(CommunityPackagesService).hasMissingPackages;
}
this.frontendSettings.mfa.enabled = isMfaFeatureEnabled();
@@ -585,9 +599,11 @@ export class Server extends AbstractServer {
}
if (config.getEnv('nodes.communityPackages.enabled')) {
controllers.push(
new NodesController(config, this.loadNodesAndCredentials, this.push, internalHooks),
// eslint-disable-next-line @typescript-eslint/naming-convention
const { CommunityPackagesController } = await import(
'@/controllers/communityPackages.controller'
);
controllers.push(Container.get(CommunityPackagesController));
}
if (inE2ETests) {
@@ -1480,9 +1496,9 @@ export class Server extends AbstractServer {
return;
}
CredentialsOverwrites().setData(body);
Container.get(CredentialsOverwrites).setData(body);
await this.loadNodesAndCredentials.generateTypesForFrontend();
await this.frontendService?.generateTypes();
this.presetCredentialsLoaded = true;
@@ -1509,22 +1525,13 @@ export class Server extends AbstractServer {
const serveIcons: express.RequestHandler = async (req, res) => {
let { scope, packageName } = req.params;
if (scope) packageName = `@${scope}/${packageName}`;
const loader = this.loadNodesAndCredentials.loaders[packageName];
if (loader) {
const pathPrefix = `/icons/${packageName}/`;
const filePath = pathResolve(
loader.directory,
req.originalUrl.substring(pathPrefix.length),
);
if (pathRelative(loader.directory, filePath).includes('..')) {
return res.status(404).end();
}
const filePath = this.loadNodesAndCredentials.resolveIcon(packageName, req.originalUrl);
if (filePath) {
try {
await fsAccess(filePath);
return res.sendFile(filePath);
} catch {}
}
res.sendStatus(404);
};

4
packages/cli/src/WorkflowRunnerProcess.ts
View File

@@ -35,8 +35,6 @@ import {
WorkflowHooks,
WorkflowOperationError,
} from 'n8n-workflow';
import { CredentialTypes } from '@/CredentialTypes';
import { CredentialsOverwrites } from '@/CredentialsOverwrites';
import * as Db from '@/Db';
import { ExternalHooks } from '@/ExternalHooks';
import type {
@@ -115,8 +113,6 @@ class WorkflowRunnerProcess {
await loadNodesAndCredentials.init();
const nodeTypes = Container.get(NodeTypes);
const credentialTypes = Container.get(CredentialTypes);
CredentialsOverwrites(credentialTypes);
// Load all external hooks
const externalHooks = Container.get(ExternalHooks);

10
packages/cli/src/audit/risks/nodes.risk.ts
View File

@@ -1,8 +1,9 @@
import * as path from 'path';
import glob from 'fast-glob';
import { Container } from 'typedi';
import config from '@/config';
import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
import { getNodeTypes } from '@/audit/utils';
import { CommunityPackageService } from '@/services/communityPackage.service';
import {
OFFICIAL_RISKY_NODE_TYPES,
ENV_VARS_DOCS_URL,
@@ -12,10 +13,13 @@ import {
} from '@/audit/constants';
import type { WorkflowEntity } from '@db/entities/WorkflowEntity';
import type { Risk } from '@/audit/types';
import { Container } from 'typedi';
async function getCommunityNodeDetails() {
const installedPackages = await Container.get(CommunityPackageService).getAllInstalledPackages();
if (!config.getEnv('nodes.communityPackages.enabled')) return [];
// eslint-disable-next-line @typescript-eslint/naming-convention
const { CommunityPackagesService } = await import('@/services/communityPackages.service');
const installedPackages = await Container.get(CommunityPackagesService).getAllInstalledPackages();
return installedPackages.reduce<Risk.CommunityNodeDetails[]>((acc, pkg) => {
pkg.installedNodes.forEach((node) =>

10
packages/cli/src/commands/BaseCommand.ts
View File

@@ -10,8 +10,6 @@ import config from '@/config';
import * as Db from '@/Db';
import * as CrashJournal from '@/CrashJournal';
import { LICENSE_FEATURES, inTest } from '@/constants';
import { CredentialTypes } from '@/CredentialTypes';
import { CredentialsOverwrites } from '@/CredentialsOverwrites';
import { initErrorHandling } from '@/ErrorReporting';
import { ExternalHooks } from '@/ExternalHooks';
import { NodeTypes } from '@/NodeTypes';
@@ -30,8 +28,6 @@ export abstract class BaseCommand extends Command {
protected externalHooks: IExternalHooksClass;
protected loadNodesAndCredentials: LoadNodesAndCredentials;
protected nodeTypes: NodeTypes;
protected userSettings: IUserSettings;
@@ -54,12 +50,8 @@ export abstract class BaseCommand extends Command {
// Make sure the settings exist
this.userSettings = await UserSettings.prepareUserSettings();
this.loadNodesAndCredentials = Container.get(LoadNodesAndCredentials);
await this.loadNodesAndCredentials.init();
await Container.get(LoadNodesAndCredentials).init();
this.nodeTypes = Container.get(NodeTypes);
this.nodeTypes.init();
const credentialTypes = Container.get(CredentialTypes);
CredentialsOverwrites(credentialTypes);
await Db.init().catch(async (error: Error) =>
this.exitWithCrash('There was an error initializing DB', error),

14
packages/cli/src/commands/start.ts
View File

@@ -23,7 +23,6 @@ import * as Db from '@/Db';
import * as GenericHelpers from '@/GenericHelpers';
import { Server } from '@/Server';
import { TestWebhooks } from '@/TestWebhooks';
import { CommunityPackageService } from '@/services/communityPackage.service';
import { EDITOR_UI_DIST_DIR, GENERATED_STATIC_DIR } from '@/constants';
import { eventBus } from '@/eventbus';
import { BaseCommand } from './BaseCommand';
@@ -257,8 +256,6 @@ export class Start extends BaseCommand {
config.set('userManagement.jwtSecret', createHash('sha256').update(baseKey).digest('hex'));
}
await this.loadNodesAndCredentials.generateTypesForFrontend();
await UserSettings.getEncryptionKey();
// Load settings from database and set them to config.
@@ -270,12 +267,11 @@ export class Start extends BaseCommand {
const areCommunityPackagesEnabled = config.getEnv('nodes.communityPackages.enabled');
if (areCommunityPackagesEnabled) {
await Container.get(CommunityPackageService).setMissingPackages(
this.loadNodesAndCredentials,
{
reinstallMissingPackages: flags.reinstallMissingPackages,
},
);
// eslint-disable-next-line @typescript-eslint/naming-convention
const { CommunityPackagesService } = await import('@/services/communityPackages.service');
await Container.get(CommunityPackagesService).setMissingPackages({
reinstallMissingPackages: flags.reinstallMissingPackages,
});
}
const dbType = config.getEnv('database.type');

2
packages/cli/src/commands/worker.ts
View File

@@ -463,7 +463,7 @@ export class Worker extends BaseCommand {
return;
}
CredentialsOverwrites().setData(body);
Container.get(CredentialsOverwrites).setData(body);
presetCredentialsLoaded = true;
ResponseHelper.sendSuccessResponse(res, { success: true }, true, 200);
} else {

6
packages/cli/src/config/schema.ts
View File

@@ -818,12 +818,6 @@ export const schema = {
env: 'N8N_COMMUNITY_PACKAGES_ENABLED',
},
},
packagesMissing: {
// Used to have a persistent list of packages
doc: 'Contains a comma separated list of packages that failed to load during startup',
format: String,
default: '',
},
},
logs: {

61
packages/cli/src/controllers/nodes.controller.ts → packages/cli/src/controllers/communityPackages.controller.ts
View File

@@ -1,4 +1,6 @@
import { Service } from 'typedi';
import { Request, Response, NextFunction } from 'express';
import config from '@/config';
import {
RESPONSE_ERROR_MESSAGES,
STARTER_TEMPLATE_NAME,
@@ -9,12 +11,9 @@ import { NodeRequest } from '@/requests';
import { BadRequestError, InternalServerError } from '@/ResponseHelper';
import type { InstalledPackages } from '@db/entities/InstalledPackages';
import type { CommunityPackages } from '@/Interfaces';
import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
import { InternalHooks } from '@/InternalHooks';
import { Push } from '@/push';
import { Config } from '@/config';
import { CommunityPackageService } from '@/services/communityPackage.service';
import Container from 'typedi';
import { CommunityPackagesService } from '@/services/communityPackages.service';
const {
PACKAGE_NOT_INSTALLED,
@@ -33,24 +32,20 @@ export function isNpmError(error: unknown): error is { code: number; stdout: str
return typeof error === 'object' && error !== null && 'code' in error && 'stdout' in error;
}
@Service()
@Authorized(['global', 'owner'])
@RestController('/nodes')
export class NodesController {
private communityPackageService: CommunityPackageService;
@RestController('/community-packages')
export class CommunityPackagesController {
constructor(
private config: Config,
private loadNodesAndCredentials: LoadNodesAndCredentials,
private push: Push,
private internalHooks: InternalHooks,
) {
this.communityPackageService = Container.get(CommunityPackageService);
}
private communityPackagesService: CommunityPackagesService,
) {}
// TODO: move this into a new decorator `@IfConfig('executions.mode', 'queue')`
@Middleware()
checkIfCommunityNodesEnabled(req: Request, res: Response, next: NextFunction) {
if (this.config.getEnv('executions.mode') === 'queue' && req.method !== 'GET')
if (config.getEnv('executions.mode') === 'queue' && req.method !== 'GET')
res.status(400).json({
status: 'error',
message: 'Package management is disabled when running in "queue" mode',
@@ -69,7 +64,7 @@ export class NodesController {
let parsed: CommunityPackages.ParsedPackageName;
try {
parsed = this.communityPackageService.parseNpmPackageName(name);
parsed = this.communityPackagesService.parseNpmPackageName(name);
} catch (error) {
throw new BadRequestError(
error instanceof Error ? error.message : 'Failed to parse package name',
@@ -85,8 +80,8 @@ export class NodesController {
);
}
const isInstalled = await this.communityPackageService.isPackageInstalled(parsed.packageName);
const hasLoaded = this.communityPackageService.hasPackageLoaded(name);
const isInstalled = await this.communityPackagesService.isPackageInstalled(parsed.packageName);
const hasLoaded = this.communityPackagesService.hasPackageLoaded(name);
if (isInstalled && hasLoaded) {
throw new BadRequestError(
@@ -97,7 +92,7 @@ export class NodesController {
);
}
const packageStatus = await this.communityPackageService.checkNpmPackageStatus(name);
const packageStatus = await this.communityPackagesService.checkNpmPackageStatus(name);
if (packageStatus.status !== 'OK') {
throw new BadRequestError(`Package "${name}" is banned so it cannot be installed`);
@@ -105,7 +100,7 @@ export class NodesController {
let installedPackage: InstalledPackages;
try {
installedPackage = await this.loadNodesAndCredentials.installNpmModule(
installedPackage = await this.communityPackagesService.installNpmModule(
parsed.packageName,
parsed.version,
);
@@ -130,7 +125,7 @@ export class NodesController {
throw new (clientError ? BadRequestError : InternalServerError)(message);
}
if (!hasLoaded) this.communityPackageService.removePackageFromMissingList(name);
if (!hasLoaded) this.communityPackagesService.removePackageFromMissingList(name);
// broadcast to connected frontends that node list has been updated
installedPackage.installedNodes.forEach((node) => {
@@ -156,7 +151,7 @@ export class NodesController {
@Get('/')
async getInstalledPackages() {
const installedPackages = await this.communityPackageService.getAllInstalledPackages();
const installedPackages = await this.communityPackagesService.getAllInstalledPackages();
if (installedPackages.length === 0) return [];
@@ -164,7 +159,7 @@ export class NodesController {
try {
const command = ['npm', 'outdated', '--json'].join(' ');
await this.communityPackageService.executeNpmCommand(command, { doNotHandleError: true });
await this.communityPackagesService.executeNpmCommand(command, { doNotHandleError: true });
} catch (error) {
// when there are updates, npm exits with code 1
// when there are no updates, command succeeds
@@ -174,18 +169,14 @@ export class NodesController {
}
}
let hydratedPackages = this.communityPackageService.matchPackagesWithUpdates(
let hydratedPackages = this.communityPackagesService.matchPackagesWithUpdates(
installedPackages,
pendingUpdates,
);
try {
const missingPackages = this.config.get('nodes.packagesMissing') as string | undefined;
if (missingPackages) {
hydratedPackages = this.communityPackageService.matchMissingPackages(
hydratedPackages,
missingPackages,
);
if (this.communityPackagesService.hasMissingPackages) {
hydratedPackages = this.communityPackagesService.matchMissingPackages(hydratedPackages);
}
} catch {}
@@ -201,21 +192,21 @@ export class NodesController {
}
try {
this.communityPackageService.parseNpmPackageName(name); // sanitize input
this.communityPackagesService.parseNpmPackageName(name); // sanitize input
} catch (error) {
const message = error instanceof Error ? error.message : UNKNOWN_FAILURE_REASON;
throw new BadRequestError(message);
}
const installedPackage = await this.communityPackageService.findInstalledPackage(name);
const installedPackage = await this.communityPackagesService.findInstalledPackage(name);
if (!installedPackage) {
throw new BadRequestError(PACKAGE_NOT_INSTALLED);
}
try {
await this.loadNodesAndCredentials.removeNpmModule(name, installedPackage);
await this.communityPackagesService.removeNpmModule(name, installedPackage);
} catch (error) {
const message = [
`Error removing package "${name}"`,
@@ -252,15 +243,15 @@ export class NodesController {
}
const previouslyInstalledPackage =
await this.communityPackageService.findInstalledPackage(name);
await this.communityPackagesService.findInstalledPackage(name);
if (!previouslyInstalledPackage) {
throw new BadRequestError(PACKAGE_NOT_INSTALLED);
}
try {
const newInstalledPackage = await this.loadNodesAndCredentials.updateNpmModule(
this.communityPackageService.parseNpmPackageName(name).packageName,
const newInstalledPackage = await this.communityPackagesService.updateNpmModule(
this.communityPackagesService.parseNpmPackageName(name).packageName,
previouslyInstalledPackage,
);

1
packages/cli/src/controllers/index.ts
View File

@@ -2,7 +2,6 @@ export { AuthController } from './auth.controller';
export { LdapController } from './ldap.controller';
export { MeController } from './me.controller';
export { MFAController } from './mfa.controller';
export { NodesController } from './nodes.controller';
export { NodeTypesController } from './nodeTypes.controller';
export { OwnerController } from './owner.controller';
export { PasswordResetController } from './passwordReset.controller';

146
packages/cli/src/services/communityPackage.service.ts → packages/cli/src/services/communityPackages.service.ts
View File

@@ -1,13 +1,14 @@
import { exec } from 'child_process';
import { access as fsAccess, mkdir as fsMkdir } from 'fs/promises';
import { LoggerProxy as Logger } from 'n8n-workflow';
import { UserSettings } from 'n8n-core';
import { Service } from 'typedi';
import { promisify } from 'util';
import axios from 'axios';
import config from '@/config';
import { LoggerProxy as Logger } from 'n8n-workflow';
import type { PublicInstalledPackage } from 'n8n-workflow';
import { UserSettings } from 'n8n-core';
import type { PackageDirectoryLoader } from 'n8n-core';
import { toError } from '@/utils';
import { InstalledPackagesRepository } from '@/databases/repositories/installedPackages.repository';
import type { InstalledPackages } from '@/databases/entities/InstalledPackages';
@@ -18,11 +19,8 @@ import {
RESPONSE_ERROR_MESSAGES,
UNKNOWN_FAILURE_REASON,
} from '@/constants';
import type { PublicInstalledPackage } from 'n8n-workflow';
import type { PackageDirectoryLoader } from 'n8n-core';
import type { CommunityPackages } from '@/Interfaces';
import type { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
const {
PACKAGE_NAME_NOT_PROVIDED,
@@ -45,8 +43,17 @@ const asyncExec = promisify(exec);
const INVALID_OR_SUSPICIOUS_PACKAGE_NAME = /[^0-9a-z@\-./]/;
@Service()
export class CommunityPackageService {
constructor(private readonly installedPackageRepository: InstalledPackagesRepository) {}
export class CommunityPackagesService {
missingPackages: string[] = [];
constructor(
private readonly installedPackageRepository: InstalledPackagesRepository,
private readonly loadNodesAndCredentials: LoadNodesAndCredentials,
) {}
get hasMissingPackages() {
return this.missingPackages.length > 0;
}
async findInstalledPackage(packageName: string) {
return this.installedPackageRepository.findOne({
@@ -173,9 +180,8 @@ export class CommunityPackageService {
}, []);
}
matchMissingPackages(installedPackages: PublicInstalledPackage[], missingPackages: string) {
const missingPackagesList = missingPackages
.split(' ')
matchMissingPackages(installedPackages: PublicInstalledPackage[]) {
const missingPackagesList = this.missingPackages
.map((name) => {
try {
// Strip away versions but maintain scope and package name
@@ -221,45 +227,34 @@ export class CommunityPackageService {
}
hasPackageLoaded(packageName: string) {
const missingPackages = config.get('nodes.packagesMissing') as string | undefined;
if (!this.missingPackages.length) return true;
if (!missingPackages) return true;
return !missingPackages
.split(' ')
.some(
(packageNameAndVersion) =>
packageNameAndVersion.startsWith(packageName) &&
packageNameAndVersion.replace(packageName, '').startsWith('@'),
);
return !this.missingPackages.some(
(packageNameAndVersion) =>
packageNameAndVersion.startsWith(packageName) &&
packageNameAndVersion.replace(packageName, '').startsWith('@'),
);
}
removePackageFromMissingList(packageName: string) {
try {
const failedPackages = config.get('nodes.packagesMissing').split(' ');
const packageFailedToLoad = failedPackages.filter(
this.missingPackages = this.missingPackages.filter(
(packageNameAndVersion) =>
!packageNameAndVersion.startsWith(packageName) ||
!packageNameAndVersion.replace(packageName, '').startsWith('@'),
);
config.set('nodes.packagesMissing', packageFailedToLoad.join(' '));
} catch {
// do nothing
}
}
async setMissingPackages(
loadNodesAndCredentials: LoadNodesAndCredentials,
{ reinstallMissingPackages }: { reinstallMissingPackages: boolean },
) {
async setMissingPackages({ reinstallMissingPackages }: { reinstallMissingPackages: boolean }) {
const installedPackages = await this.getAllInstalledPackages();
const missingPackages = new Set<{ packageName: string; version: string }>();
installedPackages.forEach((installedPackage) => {
installedPackage.installedNodes.forEach((installedNode) => {
if (!loadNodesAndCredentials.known.nodes[installedNode.type]) {
if (!this.loadNodesAndCredentials.isKnownNode(installedNode.type)) {
// Leave the list ready for installing in case we need.
missingPackages.add({
packageName: installedPackage.packageName,
@@ -269,7 +264,7 @@ export class CommunityPackageService {
});
});
config.set('nodes.packagesMissing', '');
this.missingPackages = [];
if (missingPackages.size === 0) return;
@@ -283,10 +278,7 @@ export class CommunityPackageService {
// Optimistic approach - stop if any installation fails
for (const missingPackage of missingPackages) {
await loadNodesAndCredentials.installNpmModule(
missingPackage.packageName,
missingPackage.version,
);
await this.installNpmModule(missingPackage.packageName, missingPackage.version);
missingPackages.delete(missingPackage);
}
@@ -296,11 +288,79 @@ export class CommunityPackageService {
}
}
config.set(
'nodes.packagesMissing',
Array.from(missingPackages)
.map((missingPackage) => `${missingPackage.packageName}@${missingPackage.version}`)
.join(' '),
this.missingPackages = [...missingPackages].map(
(missingPackage) => `${missingPackage.packageName}@${missingPackage.version}`,
);
}
async installNpmModule(packageName: string, version?: string): Promise<InstalledPackages> {
return this.installOrUpdateNpmModule(packageName, { version });
}
async updateNpmModule(
packageName: string,
installedPackage: InstalledPackages,
): Promise<InstalledPackages> {
return this.installOrUpdateNpmModule(packageName, { installedPackage });
}
async removeNpmModule(packageName: string, installedPackage: InstalledPackages): Promise<void> {
await this.executeNpmCommand(`npm remove ${packageName}`);
await this.removePackageFromDatabase(installedPackage);
await this.loadNodesAndCredentials.unloadPackage(packageName);
await this.loadNodesAndCredentials.postProcessLoaders();
}
private async installOrUpdateNpmModule(
packageName: string,
options: { version?: string } | { installedPackage: InstalledPackages },
) {
const isUpdate = 'installedPackage' in options;
const command = isUpdate
? `npm update ${packageName}`
: `npm install ${packageName}${options.version ? `@${options.version}` : ''}`;
try {
await this.executeNpmCommand(command);
} catch (error) {
if (error instanceof Error && error.message === RESPONSE_ERROR_MESSAGES.PACKAGE_NOT_FOUND) {
throw new Error(`The npm package "${packageName}" could not be found.`);
}
throw error;
}
let loader: PackageDirectoryLoader;
try {
loader = await this.loadNodesAndCredentials.loadPackage(packageName);
} catch (error) {
// Remove this package since loading it failed
const removeCommand = `npm remove ${packageName}`;
try {
await this.executeNpmCommand(removeCommand);
} catch {}
throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_LOADING_FAILED, { cause: error });
}
if (loader.loadedNodes.length > 0) {
// Save info to DB
try {
if (isUpdate) {
await this.removePackageFromDatabase(options.installedPackage);
}
const installedPackage = await this.persistInstalledPackage(loader);
await this.loadNodesAndCredentials.postProcessLoaders();
return installedPackage;
} catch (error) {
throw new Error(`Failed to save installed package: ${packageName}`, { cause: error });
}
} else {
// Remove this package since it contains no loadable nodes
const removeCommand = `npm remove ${packageName}`;
try {
await this.executeNpmCommand(removeCommand);
} catch {}
throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_DOES_NOT_CONTAIN_NODES);
}
}
}

67
packages/cli/src/services/frontend.service.ts Normal file
View File

@@ -0,0 +1,67 @@
import { Service } from 'typedi';
import uniq from 'lodash/uniq';
import { createWriteStream } from 'fs';
import { mkdir } from 'fs/promises';
import path from 'path';
import type { ICredentialType, INodeTypeBaseDescription } from 'n8n-workflow';
import { GENERATED_STATIC_DIR } from '@/constants';
import { CredentialsOverwrites } from '@/CredentialsOverwrites';
import { CredentialTypes } from '@/CredentialTypes';
import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
@Service()
export class FrontendService {
constructor(
private readonly loadNodesAndCredentials: LoadNodesAndCredentials,
private readonly credentialTypes: CredentialTypes,
private readonly credentialsOverwrites: CredentialsOverwrites,
) {}
async generateTypes() {
this.overwriteCredentialsProperties();
// pre-render all the node and credential types as static json files
await mkdir(path.join(GENERATED_STATIC_DIR, 'types'), { recursive: true });
const { credentials, nodes } = this.loadNodesAndCredentials.types;
this.writeStaticJSON('nodes', nodes);
this.writeStaticJSON('credentials', credentials);
}
private writeStaticJSON(name: string, data: INodeTypeBaseDescription[] | ICredentialType[]) {
const filePath = path.join(GENERATED_STATIC_DIR, `types/${name}.json`);
const stream = createWriteStream(filePath, 'utf-8');
stream.write('[\n');
data.forEach((entry, index) => {
stream.write(JSON.stringify(entry));
if (index !== data.length - 1) stream.write(',');
stream.write('\n');
});
stream.write(']\n');
stream.end();
}
private overwriteCredentialsProperties() {
const { credentials } = this.loadNodesAndCredentials.types;
const credentialsOverwrites = this.credentialsOverwrites.getAll();
for (const credential of credentials) {
const overwrittenProperties = [];
this.credentialTypes
.getParentTypes(credential.name)
.reverse()
.map((name) => credentialsOverwrites[name])
.forEach((overwrite) => {
if (overwrite) overwrittenProperties.push(...Object.keys(overwrite));
});
if (credential.name in credentialsOverwrites) {
overwrittenProperties.push(...Object.keys(credentialsOverwrites[credential.name]));
}
if (overwrittenProperties.length) {
credential.__overwrittenProperties = uniq(overwrittenProperties);
}
}
}
}