feat(core): Coordinate workflow activation in multiple main scenario in internal API (#7566)

Story: https://linear.app/n8n/issue/PAY-926

This PR coordinates workflow activation on instance startup and on
leadership change in multiple main scenario in the internal API. Part 3
on manual workflow activation and deactivation will be a separate PR.

### Part 1: Instance startup

In multi-main scenario, on starting an instance...
- [x] If the instance is the leader, it should add webhooks, triggers
and pollers.
- [x] If the instance is the follower, it should not add webhooks,
triggers or pollers.
- [x] Unit tests.

### Part 2: Leadership change 

In multi-main scenario, if the main instance leader dies…

- [x] The new main instance leader must activate all trigger- and
poller-based workflows, excluding webhook-based workflows.
- [x] The old main instance leader must deactivate all trigger- and
poller-based workflows, excluding webhook-based workflows.
- [x] Unit tests.

To test, start two instances and check behavior on startup and
leadership change:

```
EXECUTIONS_MODE=queue N8N_LEADER_SELECTION_ENABLED=true N8N_LICENSE_TENANT_ID=... N8N_LICENSE_ACTIVATION_KEY=... N8N_LOG_LEVEL=debug npm run start

EXECUTIONS_MODE=queue N8N_LEADER_SELECTION_ENABLED=true N8N_LICENSE_TENANT_ID=... N8N_LICENSE_ACTIVATION_KEY=... N8N_LOG_LEVEL=debug N8N_PORT=5679 npm run start
```
This commit is contained in:
Iván Ovejero
2023-11-07 13:48:48 +01:00
committed by GitHub
parent 151e60f829
commit c857e42677
15 changed files with 839 additions and 618 deletions

View File

@@ -2,8 +2,9 @@ import Container from 'typedi';
import { RedisService } from './redis.service';
import type { RedisServicePubSubPublisher } from './redis/RedisServicePubSubPublisher';
import config from '@/config';
import { EventEmitter } from 'node:events';
export abstract class OrchestrationService {
export abstract class OrchestrationService extends EventEmitter {
protected initialized = false;
protected queueModeId: string;
@@ -29,6 +30,7 @@ export abstract class OrchestrationService {
}
constructor() {
super();
this.redisService = Container.get(RedisService);
this.queueModeId = config.getEnv('redis.queueModeId');
}

View File

@@ -13,11 +13,11 @@ export class MultiMainInstancePublisher extends SingleMainInstancePublisher {
private leaderId: string | undefined;
public get isLeader() {
get isLeader() {
return this.id === this.leaderId;
}
public get isFollower() {
get isFollower() {
return !this.isLeader;
}
@@ -84,6 +84,8 @@ export class MultiMainInstancePublisher extends SingleMainInstancePublisher {
this.leaderId = this.id;
this.emit('leadershipChange', this.id);
await this.redisPublisher.setExpiration(this.leaderKey, this.leaderKeyTtl);
}
}