feat(core): Coordinate manual workflow activation and deactivation in multi-main scenario (#7643)

Followup to #7566 | Story: https://linear.app/n8n/issue/PAY-926

### Manual workflow activation and deactivation

In a multi-main scenario, if the user manually activates or deactivates
a workflow, the process (whether leader or follower) that handles the
PATCH request and updates its internal state should send a message into
the command channel, so that all other main processes update their
internal state accordingly:

- Add to `ActiveWorkflows` if activating
- Remove from `ActiveWorkflows` if deactivating
- Remove and re-add to `ActiveWorkflows` if the update did not change
activation status.

After updating their internal state, if activating or deactivating, the
recipient main processes should push a message to all connected
frontends so that these can update their stores and so reflect the value
in the UI.

### Workflow activation errors

On failure to activate a workflow, the main instance should record the
error in Redis - main instances should always pull activation errors
from Redis in a multi-main scenario.

### Leadership change

On leadership change...

- The old leader should stop pruning and the new leader should start
pruning.
- The old leader should remove trigger- and poller-based workflows and
the new leader should add them.

packages/cli/src/services/orchestration/main/handleCommandMessageMain.ts

@@ -5,12 +5,17 @@ import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus';
 import { ExternalSecretsManager } from '@/ExternalSecrets/ExternalSecretsManager.ee';
 import { License } from '@/License';
 import { Logger } from '@/Logger';
+import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
+import { Push } from '@/push';
+import { MultiMainSetup } from './MultiMainSetup.ee';
+import { WorkflowRepository } from '@/databases/repositories/workflow.repository';
 
 export async function handleCommandMessageMain(messageString: string) {
-	const queueModeId = config.get('redis.queueModeId');
-	const isMainInstance = config.get('generic.instanceType') === 'main';
+	const queueModeId = config.getEnv('redis.queueModeId');
+	const isMainInstance = config.getEnv('generic.instanceType') === 'main';
 	const message = messageToRedisServiceCommandObject(messageString);
 	const logger = Container.get(Logger);
+	const activeWorkflowRunner = Container.get(ActiveWorkflowRunner);
 
 	if (message) {
 		logger.debug(
@@ -35,7 +40,7 @@ export async function handleCommandMessageMain(messageString: string) {
 					return message;
 				}
 
-				if (isMainInstance && !config.getEnv('leaderSelection.enabled')) {
+				if (isMainInstance && !config.getEnv('multiMainSetup.enabled')) {
 					// at this point in time, only a single main instance is supported, thus this command _should_ never be caught currently
 					logger.error(
 						'Received command to reload license via Redis, but this should not have happened and is not supported on the main instance yet.',
@@ -60,6 +65,68 @@ export async function handleCommandMessageMain(messageString: string) {
 					return message;
 				}
 				await Container.get(ExternalSecretsManager).reloadAllProviders();
+				break;
+
+			case 'workflowActiveStateChanged': {
+				if (!debounceMessageReceiver(message, 100)) {
+					message.payload = { result: 'debounced' };
+					return message;
+				}
+
+				const { workflowId, oldState, newState, versionId } = message.payload ?? {};
+
+				if (
+					typeof workflowId !== 'string' ||
+					typeof oldState !== 'boolean' ||
+					typeof newState !== 'boolean' ||
+					typeof versionId !== 'string'
+				) {
+					break;
+				}
+
+				const push = Container.get(Push);
+
+				if (!oldState && newState) {
+					try {
+						await activeWorkflowRunner.add(workflowId, 'activate');
+						push.broadcast('workflowActivated', { workflowId });
+					} catch (e) {
+						const error = e instanceof Error ? e : new Error(`${e}`);
+
+						await Container.get(WorkflowRepository).update(workflowId, {
+							active: false,
+							versionId,
+						});
+
+						await Container.get(MultiMainSetup).broadcastWorkflowFailedToActivate({
+							workflowId,
+							errorMessage: error.message,
+						});
+					}
+				} else if (oldState && !newState) {
+					await activeWorkflowRunner.remove(workflowId);
+					push.broadcast('workflowDeactivated', { workflowId });
+				} else {
+					await activeWorkflowRunner.remove(workflowId);
+					await activeWorkflowRunner.add(workflowId, 'update');
+				}
+
+				await activeWorkflowRunner.removeActivationError(workflowId);
+			}
+
+			case 'workflowFailedToActivate': {
+				if (!debounceMessageReceiver(message, 100)) {
+					message.payload = { result: 'debounced' };
+					return message;
+				}
+
+				const { workflowId, errorMessage } = message.payload ?? {};
+
+				if (typeof workflowId !== 'string' || typeof errorMessage !== 'string') break;
+
+				Container.get(Push).broadcast('workflowFailedToActivate', { workflowId, errorMessage });
+			}
+
 			default:
 				break;
 		}