refactor(core): Have one orchestration service per instance type (#7303)

webhook instances will not listen to either worker or event log messages
on the Redis pub/sub channel

packages/cli/src/services/orchestration.base.service.ts

@@ -0,0 +1,50 @@
+import Container from 'typedi';
+import { RedisService } from './redis.service';
+import type { RedisServicePubSubPublisher } from './redis/RedisServicePubSubPublisher';
+import config from '@/config';
+
+export abstract class OrchestrationService {
+	protected initialized = false;
+
+	redisPublisher: RedisServicePubSubPublisher;
+
+	readonly redisService: RedisService;
+
+	get isQueueMode(): boolean {
+		return config.get('executions.mode') === 'queue';
+	}
+
+	get isMainInstance(): boolean {
+		return config.get('generic.instanceType') === 'main';
+	}
+
+	get isWebhookInstance(): boolean {
+		return config.get('generic.instanceType') === 'webhook';
+	}
+
+	get isWorkerInstance(): boolean {
+		return config.get('generic.instanceType') === 'worker';
+	}
+
+	constructor() {
+		this.redisService = Container.get(RedisService);
+	}
+
+	sanityCheck(): boolean {
+		return this.initialized && this.isQueueMode;
+	}
+
+	async init() {
+		await this.initPublisher();
+		this.initialized = true;
+	}
+
+	async shutdown() {
+		await this.redisPublisher?.destroy();
+		this.initialized = false;
+	}
+
+	private async initPublisher() {
+		this.redisPublisher = await this.redisService.getPubSubPublisher();
+	}
+}

packages/cli/src/services/orchestration.handler.base.service.ts

@@ -0,0 +1,33 @@
+import Container from 'typedi';
+import type { WorkerCommandReceivedHandlerOptions } from './orchestration/worker/handleCommandMessageWorker';
+import { RedisService } from './redis.service';
+import type { RedisServicePubSubSubscriber } from './redis/RedisServicePubSubSubscriber';
+
+export abstract class OrchestrationHandlerService {
+	protected initialized = false;
+
+	redisSubscriber: RedisServicePubSubSubscriber;
+
+	readonly redisService: RedisService;
+
+	constructor() {
+		this.redisService = Container.get(RedisService);
+	}
+
+	async init() {
+		await this.initSubscriber();
+		this.initialized = true;
+	}
+
+	async initWithOptions(options: WorkerCommandReceivedHandlerOptions) {
+		await this.initSubscriber(options);
+		this.initialized = true;
+	}
+
+	async shutdown() {
+		await this.redisSubscriber?.destroy();
+		this.initialized = false;
+	}
+
+	protected abstract initSubscriber(options?: WorkerCommandReceivedHandlerOptions): Promise<void>;
+}

packages/cli/src/services/orchestration.handler.service.ts

@@ -1,47 +0,0 @@
-import Container, { Service } from 'typedi';
-import { RedisService } from './redis.service';
-import type { RedisServicePubSubSubscriber } from './redis/RedisServicePubSubSubscriber';
-import {
-	COMMAND_REDIS_CHANNEL,
-	EVENT_BUS_REDIS_CHANNEL,
-	WORKER_RESPONSE_REDIS_CHANNEL,
-} from './redis/RedisServiceHelper';
-import { handleWorkerResponseMessage } from './orchestration/handleWorkerResponseMessage';
-import { handleCommandMessage } from './orchestration/handleCommandMessage';
-import { MessageEventBus } from '../eventbus/MessageEventBus/MessageEventBus';
-
-@Service()
-export class OrchestrationHandlerService {
-	redisSubscriber: RedisServicePubSubSubscriber;
-
-	constructor(readonly redisService: RedisService) {}
-
-	async init() {
-		await this.initSubscriber();
-	}
-
-	async shutdown() {
-		await this.redisSubscriber?.destroy();
-	}
-
-	private async initSubscriber() {
-		this.redisSubscriber = await this.redisService.getPubSubSubscriber();
-
-		await this.redisSubscriber.subscribeToWorkerResponseChannel();
-		await this.redisSubscriber.subscribeToCommandChannel();
-		await this.redisSubscriber.subscribeToEventLog();
-
-		this.redisSubscriber.addMessageHandler(
-			'OrchestrationMessageReceiver',
-			async (channel: string, messageString: string) => {
-				if (channel === WORKER_RESPONSE_REDIS_CHANNEL) {
-					await handleWorkerResponseMessage(messageString);
-				} else if (channel === COMMAND_REDIS_CHANNEL) {
-					await handleCommandMessage(messageString);
-				} else if (channel === EVENT_BUS_REDIS_CHANNEL) {
-					await Container.get(MessageEventBus).handleRedisEventBusMessage(messageString);
-				}
-			},
-		);
-	}
-}

packages/cli/src/services/orchestration.service.ts

@@ -1,79 +0,0 @@
-import { Service } from 'typedi';
-import { RedisService } from './redis.service';
-import type { RedisServicePubSubPublisher } from './redis/RedisServicePubSubPublisher';
-import config from '@/config';
-
-@Service()
-export class OrchestrationService {
-	private initialized = false;
-
-	redisPublisher: RedisServicePubSubPublisher;
-
-	get isQueueMode() {
-		return config.getEnv('executions.mode') === 'queue';
-	}
-
-	constructor(readonly redisService: RedisService) {}
-
-	async init() {
-		await this.initPublisher();
-		this.initialized = true;
-	}
-
-	async shutdown() {
-		await this.redisPublisher?.destroy();
-	}
-
-	private async initPublisher() {
-		this.redisPublisher = await this.redisService.getPubSubPublisher();
-	}
-
-	async getWorkerStatus(id?: string) {
-		if (!this.isQueueMode) {
-			return;
-		}
-		if (!this.initialized) {
-			throw new Error('OrchestrationService not initialized');
-		}
-		await this.redisPublisher.publishToCommandChannel({
-			command: 'getStatus',
-			targets: id ? [id] : undefined,
-		});
-	}
-
-	async getWorkerIds() {
-		if (!this.isQueueMode) {
-			return;
-		}
-		if (!this.initialized) {
-			throw new Error('OrchestrationService not initialized');
-		}
-		await this.redisPublisher.publishToCommandChannel({
-			command: 'getId',
-		});
-	}
-
-	async broadcastRestartEventbusAfterDestinationUpdate() {
-		if (!this.isQueueMode) {
-			return;
-		}
-		if (!this.initialized) {
-			throw new Error('OrchestrationService not initialized');
-		}
-		await this.redisPublisher.publishToCommandChannel({
-			command: 'restartEventBus',
-		});
-	}
-
-	async broadcastReloadExternalSecretsProviders() {
-		if (!this.isQueueMode) {
-			return;
-		}
-		if (!this.initialized) {
-			throw new Error('OrchestrationService not initialized');
-		}
-		await this.redisPublisher.publishToCommandChannel({
-			command: 'reloadExternalSecretsProviders',
-		});
-	}
-}

packages/cli/src/services/orchestration/helpers.ts

@@ -2,6 +2,10 @@ import { LoggerProxy, jsonParse } from 'n8n-workflow';
 import type { RedisServiceCommandObject } from '../redis/RedisServiceCommands';
 import { COMMAND_REDIS_CHANNEL } from '../redis/RedisServiceHelper';
 
+export interface RedisServiceCommandLastReceived {
+	[date: string]: Date;
+}
+
 export function messageToRedisServiceCommandObject(messageString: string) {
 	if (!messageString) return;
 	let message: RedisServiceCommandObject;
@@ -15,3 +19,15 @@ export function messageToRedisServiceCommandObject(messageString: string) {
 	}
 	return message;
 }
+
+const lastReceived: RedisServiceCommandLastReceived = {};
+
+export function debounceMessageReceiver(message: RedisServiceCommandObject, timeout: number = 100) {
+	const now = new Date();
+	const lastReceivedDate = lastReceived[message.command];
+	if (lastReceivedDate && now.getTime() - lastReceivedDate.getTime() < timeout) {
+		return false;
+	}
+	lastReceived[message.command] = now;
+	return true;
+}

packages/cli/src/services/orchestration/main/handleCommandMessageMain.ts

@@ -1,17 +1,14 @@
 import { LoggerProxy } from 'n8n-workflow';
-import { messageToRedisServiceCommandObject } from './helpers';
+import { debounceMessageReceiver, messageToRedisServiceCommandObject } from '../helpers';
 import config from '@/config';
 import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus';
 import Container from 'typedi';
 import { ExternalSecretsManager } from '@/ExternalSecrets/ExternalSecretsManager.ee';
-import type { N8nInstanceType } from '@/Interfaces';
 import { License } from '@/License';
 
-// this function handles commands sent to the MAIN instance. the workers handle their own commands
-export async function handleCommandMessage(messageString: string) {
+export async function handleCommandMessageMain(messageString: string) {
 	const queueModeId = config.get('redis.queueModeId');
-	const instanceType = config.get('generic.instanceType') as N8nInstanceType;
-	const isMainInstance = instanceType === 'main';
+	const isMainInstance = config.get('generic.instanceType') === 'main';
 	const message = messageToRedisServiceCommandObject(messageString);
 
 	if (message) {
@@ -30,6 +27,12 @@ export async function handleCommandMessage(messageString: string) {
 		}
 		switch (message.command) {
 			case 'reloadLicense':
+				if (!debounceMessageReceiver(message, 500)) {
+					message.payload = {
+						result: 'debounced',
+					};
+					return message;
+				}
 				if (isMainInstance) {
 					// at this point in time, only a single main instance is supported, thus this command _should_ never be caught currently
 					LoggerProxy.error(
@@ -40,8 +43,20 @@ export async function handleCommandMessage(messageString: string) {
 				await Container.get(License).reload();
 				break;
 			case 'restartEventBus':
+				if (!debounceMessageReceiver(message, 200)) {
+					message.payload = {
+						result: 'debounced',
+					};
+					return message;
+				}
 				await Container.get(MessageEventBus).restart();
 			case 'reloadExternalSecretsProviders':
+				if (!debounceMessageReceiver(message, 200)) {
+					message.payload = {
+						result: 'debounced',
+					};
+					return message;
+				}
 				await Container.get(ExternalSecretsManager).reloadAllProviders();
 			default:
 				break;

packages/cli/src/services/orchestration/main/handleWorkerResponseMessageMain.ts

@@ -1,7 +1,7 @@
 import { jsonParse, LoggerProxy } from 'n8n-workflow';
-import type { RedisServiceWorkerResponseObject } from '../redis/RedisServiceCommands';
+import type { RedisServiceWorkerResponseObject } from '../../redis/RedisServiceCommands';
 
-export async function handleWorkerResponseMessage(messageString: string) {
+export async function handleWorkerResponseMessageMain(messageString: string) {
 	const workerResponse = jsonParse<RedisServiceWorkerResponseObject>(messageString);
 	if (workerResponse) {
 		// TODO: Handle worker response

packages/cli/src/services/orchestration/main/orchestration.handler.main.service.ts

@@ -0,0 +1,34 @@
+import Container, { Service } from 'typedi';
+import {
+	COMMAND_REDIS_CHANNEL,
+	EVENT_BUS_REDIS_CHANNEL,
+	WORKER_RESPONSE_REDIS_CHANNEL,
+} from '../../redis/RedisServiceHelper';
+import { handleWorkerResponseMessageMain } from './handleWorkerResponseMessageMain';
+import { handleCommandMessageMain } from './handleCommandMessageMain';
+import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus';
+import { OrchestrationHandlerService } from '../../orchestration.handler.base.service';
+
+@Service()
+export class OrchestrationHandlerMainService extends OrchestrationHandlerService {
+	async initSubscriber() {
+		this.redisSubscriber = await this.redisService.getPubSubSubscriber();
+
+		await this.redisSubscriber.subscribeToCommandChannel();
+		await this.redisSubscriber.subscribeToWorkerResponseChannel();
+		await this.redisSubscriber.subscribeToEventLog();
+
+		this.redisSubscriber.addMessageHandler(
+			'OrchestrationMessageReceiver',
+			async (channel: string, messageString: string) => {
+				if (channel === WORKER_RESPONSE_REDIS_CHANNEL) {
+					await handleWorkerResponseMessageMain(messageString);
+				} else if (channel === COMMAND_REDIS_CHANNEL) {
+					await handleCommandMessageMain(messageString);
+				} else if (channel === EVENT_BUS_REDIS_CHANNEL) {
+					await Container.get(MessageEventBus).handleRedisEventBusMessage(messageString);
+				}
+			},
+		);
+	}
+}

packages/cli/src/services/orchestration/main/orchestration.main.service.ts

@@ -0,0 +1,38 @@
+import { Service } from 'typedi';
+import { OrchestrationService } from '../../orchestration.base.service';
+
+@Service()
+export class OrchestrationMainService extends OrchestrationService {
+	sanityCheck(): boolean {
+		return this.initialized && this.isQueueMode && this.isMainInstance;
+	}
+
+	async getWorkerStatus(id?: string) {
+		if (!this.sanityCheck()) return;
+		await this.redisPublisher.publishToCommandChannel({
+			command: 'getStatus',
+			targets: id ? [id] : undefined,
+		});
+	}
+
+	async getWorkerIds() {
+		if (!this.sanityCheck()) return;
+		await this.redisPublisher.publishToCommandChannel({
+			command: 'getId',
+		});
+	}
+
+	async broadcastRestartEventbusAfterDestinationUpdate() {
+		if (!this.sanityCheck()) return;
+		await this.redisPublisher.publishToCommandChannel({
+			command: 'restartEventBus',
+		});
+	}
+
+	async broadcastReloadExternalSecretsProviders() {
+		if (!this.sanityCheck()) return;
+		await this.redisPublisher.publishToCommandChannel({
+			command: 'reloadExternalSecretsProviders',
+		});
+	}
+}

packages/cli/src/services/orchestration/webhook/handleCommandMessageWebhook.ts

@@ -0,0 +1,6 @@
+import { handleCommandMessageMain } from '../main/handleCommandMessageMain';
+
+export async function handleCommandMessageWebhook(messageString: string) {
+	// currently webhooks handle commands the same way as the main instance
+	return handleCommandMessageMain(messageString);
+}

packages/cli/src/services/orchestration/webhook/orchestration.handler.webhook.service.ts

@@ -0,0 +1,22 @@
+import { Service } from 'typedi';
+import { COMMAND_REDIS_CHANNEL } from '../../redis/RedisServiceHelper';
+import { OrchestrationHandlerService } from '../../orchestration.handler.base.service';
+import { handleCommandMessageWebhook } from './handleCommandMessageWebhook';
+
+@Service()
+export class OrchestrationHandlerWebhookService extends OrchestrationHandlerService {
+	async initSubscriber() {
+		this.redisSubscriber = await this.redisService.getPubSubSubscriber();
+
+		await this.redisSubscriber.subscribeToCommandChannel();
+
+		this.redisSubscriber.addMessageHandler(
+			'OrchestrationMessageReceiver',
+			async (channel: string, messageString: string) => {
+				if (channel === COMMAND_REDIS_CHANNEL) {
+					await handleCommandMessageWebhook(messageString);
+				}
+			},
+		);
+	}
+}

packages/cli/src/services/orchestration/webhook/orchestration.webhook.service.ts

@@ -0,0 +1,9 @@
+import { Service } from 'typedi';
+import { OrchestrationService } from '../../orchestration.base.service';
+
+@Service()
+export class OrchestrationWebhookService extends OrchestrationService {
+	sanityCheck(): boolean {
+		return this.initialized && this.isQueueMode && this.isWebhookInstance;
+	}
+}

packages/cli/src/services/orchestration/worker/handleCommandMessageWorker.ts

@@ -0,0 +1,133 @@
+import { jsonParse, LoggerProxy } from 'n8n-workflow';
+import type { RedisServiceCommandObject } from '@/services/redis/RedisServiceCommands';
+import { COMMAND_REDIS_CHANNEL } from '@/services/redis/RedisServiceHelper';
+import type { RedisServicePubSubPublisher } from '@/services/redis/RedisServicePubSubPublisher';
+import * as os from 'os';
+import Container from 'typedi';
+import { License } from '@/License';
+import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus';
+import { ExternalSecretsManager } from '@/ExternalSecrets/ExternalSecretsManager.ee';
+import { debounceMessageReceiver } from '../helpers';
+
+export interface WorkerCommandReceivedHandlerOptions {
+	queueModeId: string;
+	instanceId: string;
+	redisPublisher: RedisServicePubSubPublisher;
+	getRunningJobIds: () => string[];
+}
+
+export function getWorkerCommandReceivedHandler(options: WorkerCommandReceivedHandlerOptions) {
+	return async (channel: string, messageString: string) => {
+		if (channel === COMMAND_REDIS_CHANNEL) {
+			if (!messageString) return;
+			let message: RedisServiceCommandObject;
+			try {
+				message = jsonParse<RedisServiceCommandObject>(messageString);
+			} catch {
+				LoggerProxy.debug(
+					`Received invalid message via channel ${COMMAND_REDIS_CHANNEL}: "${messageString}"`,
+				);
+				return;
+			}
+			if (message) {
+				LoggerProxy.debug(
+					`RedisCommandHandler(worker): Received command message ${message.command} from ${message.senderId}`,
+				);
+				if (message.targets && !message.targets.includes(options.queueModeId)) {
+					return; // early return if the message is not for this worker
+				}
+				switch (message.command) {
+					case 'getStatus':
+						if (!debounceMessageReceiver(message, 200)) return;
+						await options.redisPublisher.publishToWorkerChannel({
+							workerId: options.queueModeId,
+							command: message.command,
+							payload: {
+								workerId: options.queueModeId,
+								runningJobs: options.getRunningJobIds(),
+								freeMem: os.freemem(),
+								totalMem: os.totalmem(),
+								uptime: process.uptime(),
+								loadAvg: os.loadavg(),
+								cpus: os.cpus().map((cpu) => `${cpu.model} - speed: ${cpu.speed}`),
+								arch: os.arch(),
+								platform: os.platform(),
+								hostname: os.hostname(),
+								net: Object.values(os.networkInterfaces()).flatMap(
+									(interfaces) =>
+										interfaces?.map((net) => `${net.family} - address: ${net.address}`) ?? '',
+								),
+							},
+						});
+						break;
+					case 'getId':
+						if (!debounceMessageReceiver(message, 200)) return;
+						await options.redisPublisher.publishToWorkerChannel({
+							workerId: options.queueModeId,
+							command: message.command,
+						});
+						break;
+					case 'restartEventBus':
+						if (!debounceMessageReceiver(message, 100)) return;
+						try {
+							await Container.get(MessageEventBus).restart();
+							await options.redisPublisher.publishToWorkerChannel({
+								workerId: options.queueModeId,
+								command: message.command,
+								payload: {
+									result: 'success',
+								},
+							});
+						} catch (error) {
+							await options.redisPublisher.publishToWorkerChannel({
+								workerId: options.queueModeId,
+								command: message.command,
+								payload: {
+									result: 'error',
+									error: (error as Error).message,
+								},
+							});
+						}
+						break;
+					case 'reloadExternalSecretsProviders':
+						if (!debounceMessageReceiver(message, 200)) return;
+						try {
+							await Container.get(ExternalSecretsManager).reloadAllProviders();
+							await options.redisPublisher.publishToWorkerChannel({
+								workerId: options.queueModeId,
+								command: message.command,
+								payload: {
+									result: 'success',
+								},
+							});
+						} catch (error) {
+							await options.redisPublisher.publishToWorkerChannel({
+								workerId: options.queueModeId,
+								command: message.command,
+								payload: {
+									result: 'error',
+									error: (error as Error).message,
+								},
+							});
+						}
+						break;
+					case 'reloadLicense':
+						if (!debounceMessageReceiver(message, 500)) return;
+						await Container.get(License).reload();
+						break;
+					case 'stopWorker':
+						if (!debounceMessageReceiver(message, 500)) return;
+						// TODO: implement proper shutdown
+						// await this.stopProcess();
+						break;
+					default:
+						LoggerProxy.debug(
+							// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+							`Received unknown command via channel ${COMMAND_REDIS_CHANNEL}: "${message.command}"`,
+						);
+						break;
+				}
+			}
+		}
+	};
+}

packages/cli/src/services/orchestration/worker/orchestration.handler.worker.service.ts

@@ -0,0 +1,17 @@
+import { Service } from 'typedi';
+import { OrchestrationHandlerService } from '../../orchestration.handler.base.service';
+import type { WorkerCommandReceivedHandlerOptions } from './handleCommandMessageWorker';
+import { getWorkerCommandReceivedHandler } from './handleCommandMessageWorker';
+
+@Service()
+export class OrchestrationHandlerWorkerService extends OrchestrationHandlerService {
+	async initSubscriber(options: WorkerCommandReceivedHandlerOptions) {
+		this.redisSubscriber = await this.redisService.getPubSubSubscriber();
+
+		await this.redisSubscriber.subscribeToCommandChannel();
+		this.redisSubscriber.addMessageHandler(
+			'WorkerCommandReceivedHandler',
+			getWorkerCommandReceivedHandler(options),
+		);
+	}
+}

packages/cli/src/services/orchestration/worker/orchestration.worker.service.ts

@@ -0,0 +1,15 @@
+import { Service } from 'typedi';
+import type { AbstractEventMessage } from '@/eventbus/EventMessageClasses/AbstractEventMessage';
+import { OrchestrationService } from '../../orchestration.base.service';
+
+@Service()
+export class OrchestrationWorkerService extends OrchestrationService {
+	sanityCheck(): boolean {
+		return this.initialized && this.isQueueMode && this.isWorkerInstance;
+	}
+
+	async publishToEventLog(message: AbstractEventMessage) {
+		if (!this.sanityCheck()) return;
+		await this.redisPublisher.publishToEventLog(message);
+	}
+}