c857e42677
Story: https://linear.app/n8n/issue/PAY-926 This PR coordinates workflow activation on instance startup and on leadership change in multiple main scenario in the internal API. Part 3 on manual workflow activation and deactivation will be a separate PR. ### Part 1: Instance startup In multi-main scenario, on starting an instance... - [x] If the instance is the leader, it should add webhooks, triggers and pollers. - [x] If the instance is the follower, it should not add webhooks, triggers or pollers. - [x] Unit tests. ### Part 2: Leadership change In multi-main scenario, if the main instance leader dies… - [x] The new main instance leader must activate all trigger- and poller-based workflows, excluding webhook-based workflows. - [x] The old main instance leader must deactivate all trigger- and poller-based workflows, excluding webhook-based workflows. - [x] Unit tests. To test, start two instances and check behavior on startup and leadership change: ``` EXECUTIONS_MODE=queue N8N_LEADER_SELECTION_ENABLED=true N8N_LICENSE_TENANT_ID=... N8N_LICENSE_ACTIVATION_KEY=... N8N_LOG_LEVEL=debug npm run start EXECUTIONS_MODE=queue N8N_LEADER_SELECTION_ENABLED=true N8N_LICENSE_TENANT_ID=... N8N_LICENSE_ACTIVATION_KEY=... N8N_LOG_LEVEL=debug N8N_PORT=5679 npm run start ```
93 lines
2.3 KiB
TypeScript
93 lines
2.3 KiB
TypeScript
import config from '@/config';
|
|
import { Service } from 'typedi';
|
|
import { TIME } from '@/constants';
|
|
import { SingleMainInstancePublisher } from '@/services/orchestration/main/SingleMainInstance.publisher';
|
|
import { getRedisPrefix } from '@/services/redis/RedisServiceHelper';
|
|
|
|
/**
|
|
* For use in main instance, in multiple main instances cluster.
|
|
*/
|
|
@Service()
|
|
export class MultiMainInstancePublisher extends SingleMainInstancePublisher {
|
|
private id = this.queueModeId;
|
|
|
|
private leaderId: string | undefined;
|
|
|
|
get isLeader() {
|
|
return this.id === this.leaderId;
|
|
}
|
|
|
|
get isFollower() {
|
|
return !this.isLeader;
|
|
}
|
|
|
|
private readonly leaderKey = getRedisPrefix() + ':main_instance_leader';
|
|
|
|
private readonly leaderKeyTtl = config.getEnv('leaderSelection.ttl');
|
|
|
|
private leaderCheckInterval: NodeJS.Timer | undefined;
|
|
|
|
async init() {
|
|
if (this.initialized) return;
|
|
|
|
await this.initPublisher();
|
|
|
|
this.initialized = true;
|
|
|
|
await this.tryBecomeLeader();
|
|
|
|
this.leaderCheckInterval = setInterval(
|
|
async () => {
|
|
await this.checkLeader();
|
|
},
|
|
config.getEnv('leaderSelection.interval') * TIME.SECOND,
|
|
);
|
|
}
|
|
|
|
async destroy() {
|
|
clearInterval(this.leaderCheckInterval);
|
|
|
|
if (this.isLeader) await this.redisPublisher.clear(this.leaderKey);
|
|
}
|
|
|
|
private async checkLeader() {
|
|
if (!this.redisPublisher.redisClient) return;
|
|
|
|
const leaderId = await this.redisPublisher.get(this.leaderKey);
|
|
|
|
if (!leaderId) {
|
|
this.logger.debug('Leadership vacant, attempting to become leader...');
|
|
await this.tryBecomeLeader();
|
|
|
|
return;
|
|
}
|
|
|
|
if (this.isLeader) {
|
|
this.logger.debug(`Leader is this instance "${this.id}"`);
|
|
|
|
await this.redisPublisher.setExpiration(this.leaderKey, this.leaderKeyTtl);
|
|
} else {
|
|
this.logger.debug(`Leader is other instance "${leaderId}"`);
|
|
|
|
this.leaderId = leaderId;
|
|
}
|
|
}
|
|
|
|
private async tryBecomeLeader() {
|
|
if (this.isLeader || !this.redisPublisher.redisClient) return;
|
|
|
|
// this can only succeed if leadership is currently vacant
|
|
const keySetSuccessfully = await this.redisPublisher.setIfNotExists(this.leaderKey, this.id);
|
|
|
|
if (keySetSuccessfully) {
|
|
this.logger.debug(`Leader is now this instance "${this.id}"`);
|
|
|
|
this.leaderId = this.id;
|
|
|
|
this.emit('leadershipChange', this.id);
|
|
|
|
await this.redisPublisher.setExpiration(this.leaderKey, this.leaderKeyTtl);
|
|
}
|
|
}
|
|
}
|