feat(core): Support community packages in scaling-mode (#10228)

packages/cli/src/services/__tests__/communityPackages.service.test.ts

@@ -2,8 +2,10 @@ import { exec } from 'child_process';
 import { access as fsAccess, mkdir as fsMkdir } from 'fs/promises';
 import axios from 'axios';
 import { mocked } from 'jest-mock';
-import Container from 'typedi';
+import { mock } from 'jest-mock-extended';
+import type { GlobalConfig } from '@n8n/config';
 import type { PublicInstalledPackage } from 'n8n-workflow';
+import type { PackageDirectoryLoader } from 'n8n-core';
 
 import {
 	NODE_PACKAGE_PREFIX,
@@ -11,21 +13,18 @@ import {
 	NPM_PACKAGE_STATUS_GOOD,
 	RESPONSE_ERROR_MESSAGES,
 } from '@/constants';
-import config from '@/config';
 import { InstalledPackages } from '@db/entities/InstalledPackages';
 import type { CommunityPackages } from '@/Interfaces';
 import { CommunityPackagesService } from '@/services/communityPackages.service';
 import { InstalledNodesRepository } from '@db/repositories/installedNodes.repository';
 import { InstalledPackagesRepository } from '@db/repositories/installedPackages.repository';
 import { InstalledNodes } from '@db/entities/InstalledNodes';
-import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
+import type { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
 
 import { mockInstance } from '@test/mocking';
 import { COMMUNITY_NODE_VERSION, COMMUNITY_PACKAGE_VERSION } from '@test-integration/constants';
 import { randomName } from '@test-integration/random';
 import { mockPackageName, mockPackagePair } from '@test-integration/utils';
-import { InstanceSettings, PackageDirectoryLoader } from 'n8n-core';
-import { Logger } from '@/Logger';
 
 jest.mock('fs/promises');
 jest.mock('child_process');
@@ -40,6 +39,15 @@ const execMock = ((...args) => {
 }) as typeof exec;
 
 describe('CommunityPackagesService', () => {
+	const globalConfig = mock<GlobalConfig>({
+		nodes: {
+			communityPackages: {
+				reinstallMissing: false,
+			},
+		},
+	});
+	const loadNodesAndCredentials = mock<LoadNodesAndCredentials>();
+
 	const installedNodesRepository = mockInstance(InstalledNodesRepository);
 	installedNodesRepository.create.mockImplementation(() => {
 		const nodeName = randomName();
@@ -60,13 +68,14 @@ describe('CommunityPackagesService', () => {
 		});
 	});
 
-	mockInstance(LoadNodesAndCredentials);
-
-	const communityPackagesService = Container.get(CommunityPackagesService);
-
-	beforeEach(() => {
-		config.load(config.default);
-	});
+	const communityPackagesService = new CommunityPackagesService(
+		mock(),
+		mock(),
+		mock(),
+		loadNodesAndCredentials,
+		mock(),
+		globalConfig,
+	);
 
 	describe('parseNpmPackageName()', () => {
 		test('should fail with empty package name', () => {
@@ -365,29 +374,12 @@ describe('CommunityPackagesService', () => {
 	};
 
 	describe('updateNpmModule', () => {
-		let packageDirectoryLoader: PackageDirectoryLoader;
-		let communityPackagesService: CommunityPackagesService;
+		const packageDirectoryLoader = mock<PackageDirectoryLoader>();
 
 		beforeEach(async () => {
-			jest.restoreAllMocks();
+			jest.clearAllMocks();
 
-			packageDirectoryLoader = mockInstance(PackageDirectoryLoader);
-			const loadNodesAndCredentials = mockInstance(LoadNodesAndCredentials);
 			loadNodesAndCredentials.loadPackage.mockResolvedValue(packageDirectoryLoader);
-			const instanceSettings = mockInstance(InstanceSettings);
-			const logger = mockInstance(Logger);
-			const installedPackagesRepository = mockInstance(InstalledPackagesRepository);
-
-			communityPackagesService = new CommunityPackagesService(
-				instanceSettings,
-				logger,
-				installedPackagesRepository,
-				loadNodesAndCredentials,
-			);
-		});
-
-		afterEach(async () => {
-			jest.restoreAllMocks();
 		});
 
 		test('should call `exec` with the correct command ', async () => {
@@ -405,10 +397,7 @@ describe('CommunityPackagesService', () => {
 			//
 			// ACT
 			//
-			await communityPackagesService.updateNpmModule(
-				installedPackage.packageName,
-				installedPackage,
-			);
+			await communityPackagesService.updatePackage(installedPackage.packageName, installedPackage);
 
 			//
 			// ASSERT

packages/cli/src/services/communityPackages.service.ts

@@ -5,6 +5,7 @@ import { Service } from 'typedi';
 import { promisify } from 'util';
 import axios from 'axios';
 
+import { GlobalConfig } from '@n8n/config';
 import { ApplicationError, type PublicInstalledPackage } from 'n8n-workflow';
 import { InstanceSettings } from 'n8n-core';
 import type { PackageDirectoryLoader } from 'n8n-core';
@@ -22,6 +23,7 @@ import {
 import type { CommunityPackages } from '@/Interfaces';
 import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
 import { Logger } from '@/Logger';
+import { OrchestrationService } from './orchestration.service';
 
 const {
 	PACKAGE_NAME_NOT_PROVIDED,
@@ -45,6 +47,8 @@ const INVALID_OR_SUSPICIOUS_PACKAGE_NAME = /[^0-9a-z@\-./]/;
 
 @Service()
 export class CommunityPackagesService {
+	reinstallMissingPackages = false;
+
 	missingPackages: string[] = [];
 
 	constructor(
@@ -52,7 +56,11 @@ export class CommunityPackagesService {
 		private readonly logger: Logger,
 		private readonly installedPackageRepository: InstalledPackagesRepository,
 		private readonly loadNodesAndCredentials: LoadNodesAndCredentials,
-	) {}
+		private readonly orchestrationService: OrchestrationService,
+		globalConfig: GlobalConfig,
+	) {
+		this.reinstallMissingPackages = globalConfig.nodes.communityPackages.reinstallMissing;
+	}
 
 	get hasMissingPackages() {
 		return this.missingPackages.length > 0;
@@ -73,11 +81,11 @@ export class CommunityPackagesService {
 		return await this.installedPackageRepository.find({ relations: ['installedNodes'] });
 	}
 
-	async removePackageFromDatabase(packageName: InstalledPackages) {
+	private async removePackageFromDatabase(packageName: InstalledPackages) {
 		return await this.installedPackageRepository.remove(packageName);
 	}
 
-	async persistInstalledPackage(packageLoader: PackageDirectoryLoader) {
+	private async persistInstalledPackage(packageLoader: PackageDirectoryLoader) {
 		try {
 			return await this.installedPackageRepository.saveInstalledPackageWithNodes(packageLoader);
 		} catch (maybeError) {
@@ -251,7 +259,7 @@ export class CommunityPackagesService {
 		}
 	}
 
-	async setMissingPackages({ reinstallMissingPackages }: { reinstallMissingPackages: boolean }) {
+	async checkForMissingPackages() {
 		const installedPackages = await this.getAllInstalledPackages();
 		const missingPackages = new Set<{ packageName: string; version: string }>();
 
@@ -271,24 +279,24 @@ export class CommunityPackagesService {
 
 		if (missingPackages.size === 0) return;
 
-		this.logger.error(
-			'n8n detected that some packages are missing. For more information, visit https://docs.n8n.io/integrations/community-nodes/troubleshooting/',
-		);
-
-		if (reinstallMissingPackages || process.env.N8N_REINSTALL_MISSING_PACKAGES) {
+		if (this.reinstallMissingPackages) {
 			this.logger.info('Attempting to reinstall missing packages', { missingPackages });
 			try {
 				// Optimistic approach - stop if any installation fails
-
 				for (const missingPackage of missingPackages) {
-					await this.installNpmModule(missingPackage.packageName, missingPackage.version);
+					await this.installPackage(missingPackage.packageName, missingPackage.version);
 
 					missingPackages.delete(missingPackage);
 				}
 				this.logger.info('Packages reinstalled successfully. Resuming regular initialization.');
+				await this.loadNodesAndCredentials.postProcessLoaders();
 			} catch (error) {
 				this.logger.error('n8n was unable to install the missing packages.');
 			}
+		} else {
+			this.logger.warn(
+				'n8n detected that some packages are missing. For more information, visit https://docs.n8n.io/integrations/community-nodes/troubleshooting/',
+			);
 		}
 
 		this.missingPackages = [...missingPackages].map(
@@ -296,32 +304,30 @@ export class CommunityPackagesService {
 		);
 	}
 
-	async installNpmModule(packageName: string, version?: string): Promise<InstalledPackages> {
-		return await this.installOrUpdateNpmModule(packageName, { version });
+	async installPackage(packageName: string, version?: string): Promise<InstalledPackages> {
+		return await this.installOrUpdatePackage(packageName, { version });
 	}
 
-	async updateNpmModule(
+	async updatePackage(
 		packageName: string,
 		installedPackage: InstalledPackages,
 	): Promise<InstalledPackages> {
-		return await this.installOrUpdateNpmModule(packageName, { installedPackage });
+		return await this.installOrUpdatePackage(packageName, { installedPackage });
 	}
 
-	async removeNpmModule(packageName: string, installedPackage: InstalledPackages): Promise<void> {
-		await this.executeNpmCommand(`npm remove ${packageName}`);
+	async removePackage(packageName: string, installedPackage: InstalledPackages): Promise<void> {
+		await this.removeNpmPackage(packageName);
 		await this.removePackageFromDatabase(installedPackage);
-		await this.loadNodesAndCredentials.unloadPackage(packageName);
-		await this.loadNodesAndCredentials.postProcessLoaders();
+		await this.orchestrationService.publish('community-package-uninstall', { packageName });
 	}
 
-	private async installOrUpdateNpmModule(
+	private async installOrUpdatePackage(
 		packageName: string,
 		options: { version?: string } | { installedPackage: InstalledPackages },
 	) {
 		const isUpdate = 'installedPackage' in options;
-		const command = isUpdate
-			? `npm install ${packageName}@latest`
-			: `npm install ${packageName}${options.version ? `@${options.version}` : ''}`;
+		const packageVersion = isUpdate || !options.version ? 'latest' : options.version;
+		const command = `npm install ${packageName}@${packageVersion}`;
 
 		try {
 			await this.executeNpmCommand(command);
@@ -337,9 +343,8 @@ export class CommunityPackagesService {
 			loader = await this.loadNodesAndCredentials.loadPackage(packageName);
 		} catch (error) {
 			// Remove this package since loading it failed
-			const removeCommand = `npm remove ${packageName}`;
 			try {
-				await this.executeNpmCommand(removeCommand);
+				await this.executeNpmCommand(`npm remove ${packageName}`);
 			} catch {}
 			throw new ApplicationError(RESPONSE_ERROR_MESSAGES.PACKAGE_LOADING_FAILED, { cause: error });
 		}
@@ -351,7 +356,12 @@ export class CommunityPackagesService {
 					await this.removePackageFromDatabase(options.installedPackage);
 				}
 				const installedPackage = await this.persistInstalledPackage(loader);
+				await this.orchestrationService.publish(
+					isUpdate ? 'community-package-update' : 'community-package-install',
+					{ packageName, packageVersion },
+				);
 				await this.loadNodesAndCredentials.postProcessLoaders();
+				this.logger.info(`Community package installed: ${packageName}`);
 				return installedPackage;
 			} catch (error) {
 				throw new ApplicationError('Failed to save installed package', {
@@ -361,12 +371,24 @@ export class CommunityPackagesService {
 			}
 		} else {
 			// Remove this package since it contains no loadable nodes
-			const removeCommand = `npm remove ${packageName}`;
 			try {
-				await this.executeNpmCommand(removeCommand);
+				await this.executeNpmCommand(`npm remove ${packageName}`);
 			} catch {}
-
 			throw new ApplicationError(RESPONSE_ERROR_MESSAGES.PACKAGE_DOES_NOT_CONTAIN_NODES);
 		}
 	}
+
+	async installOrUpdateNpmPackage(packageName: string, packageVersion: string) {
+		await this.executeNpmCommand(`npm install ${packageName}@${packageVersion}`);
+		await this.loadNodesAndCredentials.loadPackage(packageName);
+		await this.loadNodesAndCredentials.postProcessLoaders();
+		this.logger.info(`Community package installed: ${packageName}`);
+	}
+
+	async removeNpmPackage(packageName: string) {
+		await this.executeNpmCommand(`npm remove ${packageName}`);
+		await this.loadNodesAndCredentials.unloadPackage(packageName);
+		await this.loadNodesAndCredentials.postProcessLoaders();
+		this.logger.info(`Community package uninstalled: ${packageName}`);
+	}
 }

packages/cli/src/services/orchestration/main/handleCommandMessageMain.ts

@@ -10,6 +10,7 @@ import { Push } from '@/push';
 import { TestWebhooks } from '@/TestWebhooks';
 import { OrchestrationService } from '@/services/orchestration.service';
 import { WorkflowRepository } from '@/databases/repositories/workflow.repository';
+import { CommunityPackagesService } from '@/services/communityPackages.service';
 
 // eslint-disable-next-line complexity
 export async function handleCommandMessageMain(messageString: string) {
@@ -77,6 +78,20 @@ export async function handleCommandMessageMain(messageString: string) {
 				}
 				await Container.get(ExternalSecretsManager).reloadAllProviders();
 				break;
+			case 'community-package-install':
+			case 'community-package-update':
+			case 'community-package-uninstall':
+				if (!debounceMessageReceiver(message, 200)) {
+					return message;
+				}
+				const { packageName, packageVersion } = message.payload;
+				const communityPackagesService = Container.get(CommunityPackagesService);
+				if (message.command === 'community-package-uninstall') {
+					await communityPackagesService.removeNpmPackage(packageName);
+				} else {
+					await communityPackagesService.installOrUpdateNpmPackage(packageName, packageVersion);
+				}
+				break;
 
 			case 'add-webhooks-triggers-and-pollers': {
 				if (!debounceMessageReceiver(message, 100)) {

packages/cli/src/services/orchestration/webhook/handleCommandMessageWebhook.ts

@@ -5,6 +5,7 @@ import Container from 'typedi';
 import { Logger } from 'winston';
 import { messageToRedisServiceCommandObject, debounceMessageReceiver } from '../helpers';
 import config from '@/config';
+import { CommunityPackagesService } from '@/services/communityPackages.service';
 
 export async function handleCommandMessageWebhook(messageString: string) {
 	const queueModeId = config.getEnv('redis.queueModeId');
@@ -63,6 +64,20 @@ export async function handleCommandMessageWebhook(messageString: string) {
 				}
 				await Container.get(ExternalSecretsManager).reloadAllProviders();
 				break;
+			case 'community-package-install':
+			case 'community-package-update':
+			case 'community-package-uninstall':
+				if (!debounceMessageReceiver(message, 200)) {
+					return message;
+				}
+				const { packageName, packageVersion } = message.payload;
+				const communityPackagesService = Container.get(CommunityPackagesService);
+				if (message.command === 'community-package-uninstall') {
+					await communityPackagesService.removeNpmPackage(packageName);
+				} else {
+					await communityPackagesService.installOrUpdateNpmPackage(packageName, packageVersion);
+				}
+				break;
 
 			default:
 				break;

packages/cli/src/services/orchestration/worker/handleCommandMessageWorker.ts

@@ -10,6 +10,7 @@ import { debounceMessageReceiver, getOsCpuString } from '../helpers';
 import type { WorkerCommandReceivedHandlerOptions } from './types';
 import { Logger } from '@/Logger';
 import { N8N_VERSION } from '@/constants';
+import { CommunityPackagesService } from '@/services/communityPackages.service';
 
 export function getWorkerCommandReceivedHandler(options: WorkerCommandReceivedHandlerOptions) {
 	// eslint-disable-next-line complexity
@@ -112,6 +113,18 @@ export function getWorkerCommandReceivedHandler(options: WorkerCommandReceivedHa
 							});
 						}
 						break;
+					case 'community-package-install':
+					case 'community-package-update':
+					case 'community-package-uninstall':
+						if (!debounceMessageReceiver(message, 500)) return;
+						const { packageName, packageVersion } = message.payload;
+						const communityPackagesService = Container.get(CommunityPackagesService);
+						if (message.command === 'community-package-uninstall') {
+							await communityPackagesService.removeNpmPackage(packageName);
+						} else {
+							await communityPackagesService.installOrUpdateNpmPackage(packageName, packageVersion);
+						}
+						break;
 					case 'reloadLicense':
 						if (!debounceMessageReceiver(message, 500)) return;
 						await Container.get(License).reload();

packages/cli/src/services/redis/RedisServiceCommands.ts

@@ -7,6 +7,9 @@ export type RedisServiceCommand =
 	| 'stopWorker'
 	| 'reloadLicense'
 	| 'reloadExternalSecretsProviders'
+	| 'community-package-install'
+	| 'community-package-update'
+	| 'community-package-uninstall'
 	| 'display-workflow-activation' // multi-main only
 	| 'display-workflow-deactivation' // multi-main only
 	| 'add-webhooks-triggers-and-pollers' // multi-main only
@@ -26,7 +29,11 @@ export type RedisServiceBaseCommand =
 			senderId: string;
 			command: Exclude<
 				RedisServiceCommand,
-				'relay-execution-lifecycle-event' | 'clear-test-webhooks'
+				| 'relay-execution-lifecycle-event'
+				| 'clear-test-webhooks'
+				| 'community-package-install'
+				| 'community-package-update'
+				| 'community-package-uninstall'
 			>;
 			payload?: {
 				[key: string]: string | number | boolean | string[] | number[] | boolean[];
@@ -41,6 +48,14 @@ export type RedisServiceBaseCommand =
 			senderId: string;
 			command: 'clear-test-webhooks';
 			payload: { webhookKey: string; workflowEntity: IWorkflowDb; pushRef: string };
+	  }
+	| {
+			senderId: string;
+			command:
+				| 'community-package-install'
+				| 'community-package-update'
+				| 'community-package-uninstall';
+			payload: { packageName: string; packageVersion: string };
 	  };
 
 export type RedisServiceWorkerResponseObject = {