feat(core): Add commands to workers to respond with current state (#7029)

This PR adds new endpoints to the REST API:
`/orchestration/worker/status` and `/orchestration/worker/id`

Currently these just trigger the return of status / ids from the workers
via the redis back channel, this still needs to be handled and passed
through to the frontend.

It also adds the eventbus to each worker, and triggers a reload of those
eventbus instances when the configuration changes on the main instances.

packages/cli/src/services/orchestration.service.ts

@@ -0,0 +1,172 @@
+import { Service } from 'typedi';
+import { RedisService } from './redis.service';
+import type { RedisServicePubSubPublisher } from './redis/RedisServicePubSubPublisher';
+import type { RedisServicePubSubSubscriber } from './redis/RedisServicePubSubSubscriber';
+import { LoggerProxy, jsonParse } from 'n8n-workflow';
+import { eventBus } from '../eventbus';
+import type { AbstractEventMessageOptions } from '../eventbus/EventMessageClasses/AbstractEventMessageOptions';
+import { getEventMessageObjectByType } from '../eventbus/EventMessageClasses/Helpers';
+import type {
+	RedisServiceCommandObject,
+	RedisServiceWorkerResponseObject,
+} from './redis/RedisServiceCommands';
+import {
+	COMMAND_REDIS_CHANNEL,
+	EVENT_BUS_REDIS_CHANNEL,
+	WORKER_RESPONSE_REDIS_CHANNEL,
+} from './redis/RedisServiceHelper';
+
+@Service()
+export class OrchestrationService {
+	private initialized = false;
+
+	private _uniqueInstanceId = '';
+
+	get uniqueInstanceId(): string {
+		return this._uniqueInstanceId;
+	}
+
+	redisPublisher: RedisServicePubSubPublisher;
+
+	redisSubscriber: RedisServicePubSubSubscriber;
+
+	constructor(readonly redisService: RedisService) {}
+
+	async init(uniqueInstanceId: string) {
+		this._uniqueInstanceId = uniqueInstanceId;
+		await this.initPublisher();
+		await this.initSubscriber();
+		this.initialized = true;
+	}
+
+	async shutdown() {
+		await this.redisPublisher?.destroy();
+		await this.redisSubscriber?.destroy();
+	}
+
+	private async initPublisher() {
+		this.redisPublisher = await this.redisService.getPubSubPublisher();
+	}
+
+	private async initSubscriber() {
+		this.redisSubscriber = await this.redisService.getPubSubSubscriber();
+
+		// TODO: these are all proof of concept implementations for the moment
+		// until worker communication is implemented
+		// #region proof of concept
+		await this.redisSubscriber.subscribeToEventLog();
+		await this.redisSubscriber.subscribeToWorkerResponseChannel();
+		await this.redisSubscriber.subscribeToCommandChannel();
+
+		this.redisSubscriber.addMessageHandler(
+			'OrchestrationMessageReceiver',
+			async (channel: string, messageString: string) => {
+				// TODO: this is a proof of concept implementation to forward events to the main instance's event bus
+				// Events are arriving through a pub/sub channel and are forwarded to the eventBus
+				// In the future, a stream should probably replace this implementation entirely
+				if (channel === EVENT_BUS_REDIS_CHANNEL) {
+					await this.handleEventBusMessage(messageString);
+				} else if (channel === WORKER_RESPONSE_REDIS_CHANNEL) {
+					await this.handleWorkerResponseMessage(messageString);
+				} else if (channel === COMMAND_REDIS_CHANNEL) {
+					await this.handleCommandMessage(messageString);
+				}
+			},
+		);
+	}
+
+	async handleWorkerResponseMessage(messageString: string) {
+		const workerResponse = jsonParse<RedisServiceWorkerResponseObject>(messageString);
+		if (workerResponse) {
+			// TODO: Handle worker response
+			LoggerProxy.debug('Received worker response', workerResponse);
+		}
+		return workerResponse;
+	}
+
+	async handleEventBusMessage(messageString: string) {
+		const eventData = jsonParse<AbstractEventMessageOptions>(messageString);
+		if (eventData) {
+			const eventMessage = getEventMessageObjectByType(eventData);
+			if (eventMessage) {
+				await eventBus.send(eventMessage);
+			}
+		}
+		return eventData;
+	}
+
+	async handleCommandMessage(messageString: string) {
+		if (!messageString) return;
+		let message: RedisServiceCommandObject;
+		try {
+			message = jsonParse<RedisServiceCommandObject>(messageString);
+		} catch {
+			LoggerProxy.debug(
+				`Received invalid message via channel ${COMMAND_REDIS_CHANNEL}: "${messageString}"`,
+			);
+			return;
+		}
+		if (message) {
+			if (
+				message.senderId === this.uniqueInstanceId ||
+				(message.targets && !message.targets.includes(this.uniqueInstanceId))
+			) {
+				LoggerProxy.debug(
+					`Skipping command message ${message.command} because it's not for this instance.`,
+				);
+				return message;
+			}
+			switch (message.command) {
+				case 'restartEventBus':
+					await eventBus.restart();
+					break;
+			}
+			return message;
+		}
+		return;
+	}
+
+	async getWorkerStatus(id?: string) {
+		if (!this.initialized) {
+			throw new Error('OrchestrationService not initialized');
+		}
+		await this.redisPublisher.publishToCommandChannel({
+			senderId: this.uniqueInstanceId,
+			command: 'getStatus',
+			targets: id ? [id] : undefined,
+		});
+	}
+
+	async getWorkerIds() {
+		if (!this.initialized) {
+			throw new Error('OrchestrationService not initialized');
+		}
+		await this.redisPublisher.publishToCommandChannel({
+			senderId: this.uniqueInstanceId,
+			command: 'getId',
+		});
+	}
+
+	// TODO: not implemented yet on worker side
+	async stopWorker(id?: string) {
+		if (!this.initialized) {
+			throw new Error('OrchestrationService not initialized');
+		}
+		await this.redisPublisher.publishToCommandChannel({
+			senderId: this.uniqueInstanceId,
+			command: 'stopWorker',
+			targets: id ? [id] : undefined,
+		});
+	}
+
+	async restartEventBus(id?: string) {
+		if (!this.initialized) {
+			throw new Error('OrchestrationService not initialized');
+		}
+		await this.redisPublisher.publishToCommandChannel({
+			senderId: this.uniqueInstanceId,
+			command: 'restartEventBus',
+			targets: id ? [id] : undefined,
+		});
+	}
+}

packages/cli/src/services/redis/RedisServiceCommands.ts

@@ -1,12 +1,13 @@
-export type RedisServiceCommand = 'getStatus' | 'restartEventBus' | 'stopWorker'; // TODO: add more commands
+export type RedisServiceCommand = 'getStatus' | 'getId' | 'restartEventBus' | 'stopWorker'; // TODO: add more commands
 
 /**
  * An object to be sent via Redis pub/sub from the main process to the workers.
  * @field command: The command to be executed.
  * @field targets: The targets to execute the command on. Leave empty to execute on all workers or specify worker ids.
- * @field args: Optional arguments to be passed to the command.
+ * @field payload: Optional arguments to be sent with the command.
  */
 type RedisServiceBaseCommand = {
+	senderId: string;
 	command: RedisServiceCommand;
 	payload?: {
 		[key: string]: string | number | boolean | string[] | number[] | boolean[];
@@ -15,7 +16,38 @@ type RedisServiceBaseCommand = {
 
 export type RedisServiceWorkerResponseObject = {
 	workerId: string;
-} & RedisServiceBaseCommand;
+} & (
+	| RedisServiceBaseCommand
+	| {
+			command: 'getStatus';
+			payload: {
+				workerId: string;
+				runningJobs: string[];
+				freeMem: number;
+				totalMem: number;
+				uptime: number;
+				loadAvg: number[];
+				cpus: string[];
+				arch: string;
+				platform: NodeJS.Platform;
+				hostname: string;
+				net: string[];
+			};
+	  }
+	| {
+			command: 'getId';
+	  }
+	| {
+			command: 'restartEventBus';
+			payload: {
+				result: 'success' | 'error';
+				error?: string;
+			};
+	  }
+	| {
+			command: 'stopWorker';
+	  }
+);
 
 export type RedisServiceCommandObject = {
 	targets?: string[];

packages/cli/src/services/redis/RedisServicePubSubSubscriber.ts

@@ -23,11 +23,11 @@ export class RedisServicePubSubSubscriber extends RedisServiceBaseReceiver {
 		if (!this.redisClient) {
 			await this.init();
 		}
-		await this.redisClient?.subscribe(channel, (error, count: number) => {
+		await this.redisClient?.subscribe(channel, (error, _count: number) => {
 			if (error) {
 				Logger.error(`Error subscribing to channel ${channel}`);
 			} else {
-				Logger.debug(`Subscribed ${count.toString()} to eventlog channel`);
+				Logger.debug(`Subscribed Redis PubSub client to channel: ${channel}`);
 			}
 		});
 	}