refactor(core): Have one orchestration service per instance type (#7303)

webhook instances will not listen to either worker or event log messages
on the Redis pub/sub channel

packages/cli/src/commands/start.ts

@@ -31,6 +31,8 @@ import { InternalHooks } from '@/InternalHooks';
 import { License } from '@/License';
 import { ExecutionRepository } from '@/databases/repositories/execution.repository';
 import { IConfig } from '@oclif/config';
+import { OrchestrationMainService } from '@/services/orchestration/main/orchestration.main.service';
+import { OrchestrationHandlerMainService } from '@/services/orchestration/main/orchestration.handler.main.service';
 
 // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-var-requires
 const open = require('open');
@@ -214,6 +216,8 @@ export class Start extends BaseCommand {
 
 		await this.initLicense();
 		this.logger.debug('License init complete');
+		await this.initOrchestration();
+		this.logger.debug('Orchestration init complete');
 		await this.initBinaryDataService();
 		this.logger.debug('Binary data service init complete');
 		await this.initExternalHooks();
@@ -228,6 +232,13 @@ export class Start extends BaseCommand {
 		}
 	}
 
+	async initOrchestration() {
+		if (config.get('executions.mode') === 'queue') {
+			await Container.get(OrchestrationMainService).init();
+			await Container.get(OrchestrationHandlerMainService).init();
+		}
+	}
+
 	async run() {
 		// eslint-disable-next-line @typescript-eslint/no-shadow
 		const { flags } = this.parse(Start);

packages/cli/src/commands/webhook.ts

@@ -7,6 +7,8 @@ import { Queue } from '@/Queue';
 import { BaseCommand } from './BaseCommand';
 import { Container } from 'typedi';
 import { IConfig } from '@oclif/config';
+import { OrchestrationWebhookService } from '@/services/orchestration/webhook/orchestration.webhook.service';
+import { OrchestrationHandlerWebhookService } from '@/services/orchestration/webhook/orchestration.handler.webhook.service';
 
 export class Webhook extends BaseCommand {
 	static description = 'Starts n8n webhook process. Intercepts only production URLs.';
@@ -94,6 +96,8 @@ export class Webhook extends BaseCommand {
 
 		await this.initLicense();
 		this.logger.debug('License init complete');
+		await this.initOrchestration();
+		this.logger.debug('Orchestration init complete');
 		await this.initBinaryDataService();
 		this.logger.debug('Binary data service init complete');
 		await this.initExternalHooks();
@@ -115,4 +119,9 @@ export class Webhook extends BaseCommand {
 	async catch(error: Error) {
 		await this.exitWithCrash('Exiting due to an error.', error);
 	}
+
+	async initOrchestration() {
+		await Container.get(OrchestrationWebhookService).init();
+		await Container.get(OrchestrationHandlerWebhookService).init();
+	}
 }

packages/cli/src/commands/worker.ts

@@ -32,12 +32,12 @@ import { OwnershipService } from '@/services/ownership.service';
 import type { ICredentialsOverwrite } from '@/Interfaces';
 import { CredentialsOverwrites } from '@/CredentialsOverwrites';
 import { rawBodyReader, bodyParser } from '@/middlewares';
-import { eventBus } from '../eventbus';
-import { RedisServicePubSubPublisher } from '../services/redis/RedisServicePubSubPublisher';
-import { RedisServicePubSubSubscriber } from '../services/redis/RedisServicePubSubSubscriber';
-import { EventMessageGeneric } from '../eventbus/EventMessageClasses/EventMessageGeneric';
-import { getWorkerCommandReceivedHandler } from '../worker/workerCommandHandler';
+import { eventBus } from '@/eventbus';
+import type { RedisServicePubSubSubscriber } from '@/services/redis/RedisServicePubSubSubscriber';
+import { EventMessageGeneric } from '@/eventbus/EventMessageClasses/EventMessageGeneric';
 import { IConfig } from '@oclif/config';
+import { OrchestrationHandlerWorkerService } from '@/services/orchestration/worker/orchestration.handler.worker.service';
+import { OrchestrationWorkerService } from '@/services/orchestration/worker/orchestration.worker.service';
 
 export class Worker extends BaseCommand {
 	static description = '\nStarts a n8n worker';
@@ -58,8 +58,6 @@ export class Worker extends BaseCommand {
 
 	static jobQueue: JobQueue;
 
-	redisPublisher: RedisServicePubSubPublisher;
-
 	redisSubscriber: RedisServicePubSubSubscriber;
 
 	/**
@@ -272,10 +270,20 @@ export class Worker extends BaseCommand {
 		this.logger.debug('External secrets init complete');
 		await this.initEventBus();
 		this.logger.debug('Event bus init complete');
-		await this.initRedis();
-		this.logger.debug('Redis init complete');
 		await this.initQueue();
 		this.logger.debug('Queue init complete');
+		await this.initOrchestration();
+		this.logger.debug('Orchestration init complete');
+		await this.initQueue();
+
+		await Container.get(OrchestrationWorkerService).publishToEventLog(
+			new EventMessageGeneric({
+				eventName: 'n8n.worker.started',
+				payload: {
+					workerId: this.queueModeId,
+				},
+			}),
+		);
 	}
 
 	async initEventBus() {
@@ -290,29 +298,14 @@ export class Worker extends BaseCommand {
 	 * A subscription connection to redis is created to subscribe to commands from the main process
 	 * The subscription connection adds a handler to handle the command messages
 	 */
-	async initRedis() {
-		this.redisPublisher = Container.get(RedisServicePubSubPublisher);
-		this.redisSubscriber = Container.get(RedisServicePubSubSubscriber);
-		await this.redisPublisher.init();
-		await this.redisPublisher.publishToEventLog(
-			new EventMessageGeneric({
-				eventName: 'n8n.worker.started',
-				payload: {
-					workerId: this.queueModeId,
-				},
-			}),
-		);
-		await this.redisSubscriber.subscribeToCommandChannel();
-		this.redisSubscriber.addMessageHandler(
-			'WorkerCommandReceivedHandler',
-			// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
-			getWorkerCommandReceivedHandler({
-				queueModeId: this.queueModeId,
-				instanceId: this.instanceId,
-				redisPublisher: this.redisPublisher,
-				getRunningJobIds: () => Object.keys(Worker.runningJobs),
-			}),
-		);
+	async initOrchestration() {
+		await Container.get(OrchestrationWorkerService).init();
+		await Container.get(OrchestrationHandlerWorkerService).initWithOptions({
+			queueModeId: this.queueModeId,
+			instanceId: this.instanceId,
+			redisPublisher: Container.get(OrchestrationWorkerService).redisPublisher,
+			getRunningJobIds: () => Object.keys(Worker.runningJobs),
+		});
 	}
 
 	async initQueue() {