feat(core): Coordinate workflow activation in multiple main scenario in internal API (#7566)
Story: https://linear.app/n8n/issue/PAY-926 This PR coordinates workflow activation on instance startup and on leadership change in multiple main scenario in the internal API. Part 3 on manual workflow activation and deactivation will be a separate PR. ### Part 1: Instance startup In multi-main scenario, on starting an instance... - [x] If the instance is the leader, it should add webhooks, triggers and pollers. - [x] If the instance is the follower, it should not add webhooks, triggers or pollers. - [x] Unit tests. ### Part 2: Leadership change In multi-main scenario, if the main instance leader dies… - [x] The new main instance leader must activate all trigger- and poller-based workflows, excluding webhook-based workflows. - [x] The old main instance leader must deactivate all trigger- and poller-based workflows, excluding webhook-based workflows. - [x] Unit tests. To test, start two instances and check behavior on startup and leadership change: ``` EXECUTIONS_MODE=queue N8N_LEADER_SELECTION_ENABLED=true N8N_LICENSE_TENANT_ID=... N8N_LICENSE_ACTIVATION_KEY=... N8N_LOG_LEVEL=debug npm run start EXECUTIONS_MODE=queue N8N_LEADER_SELECTION_ENABLED=true N8N_LICENSE_TENANT_ID=... N8N_LICENSE_ACTIVATION_KEY=... N8N_LOG_LEVEL=debug N8N_PORT=5679 npm run start ```
This commit is contained in:
Iván Ovejero
GitHub
@@ -1,278 +0,0 @@
|
||||
import { v4 as uuid } from 'uuid';
|
||||
import { mocked } from 'jest-mock';
|
||||
import { Container } from 'typedi';
|
||||
|
||||
import type { INode } from 'n8n-workflow';
|
||||
import { NodeApiError, NodeOperationError, Workflow } from 'n8n-workflow';
|
||||
|
||||
import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
|
||||
import * as Db from '@/Db';
|
||||
import { WorkflowEntity } from '@db/entities/WorkflowEntity';
|
||||
import { SharedWorkflow } from '@db/entities/SharedWorkflow';
|
||||
import { Role } from '@db/entities/Role';
|
||||
import { User } from '@db/entities/User';
|
||||
import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData';
|
||||
import { WorkflowRunner } from '@/WorkflowRunner';
|
||||
import { ExternalHooks } from '@/ExternalHooks';
|
||||
import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
|
||||
import { Push } from '@/push';
|
||||
import { ActiveExecutions } from '@/ActiveExecutions';
|
||||
import { SecretsHelper } from '@/SecretsHelpers';
|
||||
import { WebhookService } from '@/services/webhook.service';
|
||||
import { VariablesService } from '@/environments/variables/variables.service';
|
||||
|
||||
import { mockInstance } from '../integration/shared/utils/';
|
||||
import { randomEmail, randomName } from '../integration/shared/random';
|
||||
import * as Helpers from './Helpers';
|
||||
|
||||
/**
|
||||
* TODO:
|
||||
* - test workflow webhooks activation (that trigger `executeWebhook`and other webhook methods)
|
||||
* - test activation error catching and getters such as `getActivationError` (requires building a workflow that fails to activate)
|
||||
* - test queued workflow activation functions (might need to create a non-working workflow to test this)
|
||||
*/
|
||||
|
||||
let databaseActiveWorkflowsCount = 0;
|
||||
let databaseActiveWorkflowsList: WorkflowEntity[] = [];
|
||||
|
||||
const generateWorkflows = (count: number): WorkflowEntity[] => {
|
||||
const workflows: WorkflowEntity[] = [];
|
||||
const ownerRole = new Role();
|
||||
ownerRole.scope = 'workflow';
|
||||
ownerRole.name = 'owner';
|
||||
ownerRole.id = '1';
|
||||
|
||||
const owner = new User();
|
||||
owner.id = uuid();
|
||||
owner.firstName = randomName();
|
||||
owner.lastName = randomName();
|
||||
owner.email = randomEmail();
|
||||
|
||||
for (let i = 0; i < count; i++) {
|
||||
const workflow = new WorkflowEntity();
|
||||
Object.assign(workflow, {
|
||||
id: (i + 1).toString(),
|
||||
name: randomName(),
|
||||
active: true,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
nodes: [
|
||||
{
|
||||
parameters: {
|
||||
rule: {
|
||||
interval: [{}],
|
||||
},
|
||||
},
|
||||
id: uuid(),
|
||||
name: 'Schedule Trigger',
|
||||
type: 'n8n-nodes-base.scheduleTrigger',
|
||||
typeVersion: 1,
|
||||
position: [900, 460],
|
||||
},
|
||||
],
|
||||
connections: {},
|
||||
tags: [],
|
||||
});
|
||||
const sharedWorkflow = new SharedWorkflow();
|
||||
sharedWorkflow.workflowId = workflow.id;
|
||||
sharedWorkflow.role = ownerRole;
|
||||
sharedWorkflow.user = owner;
|
||||
|
||||
workflow.shared = [sharedWorkflow];
|
||||
|
||||
workflows.push(workflow);
|
||||
}
|
||||
databaseActiveWorkflowsList = workflows;
|
||||
return workflows;
|
||||
};
|
||||
|
||||
const MOCK_NODE_TYPES_DATA = Helpers.mockNodeTypesData(['scheduleTrigger'], {
|
||||
addTrigger: true,
|
||||
});
|
||||
|
||||
jest.mock('@/Db', () => {
|
||||
return {
|
||||
collections: {
|
||||
Workflow: {
|
||||
find: jest.fn(async () => generateWorkflows(databaseActiveWorkflowsCount)),
|
||||
findOne: jest.fn(async (searchParams) => {
|
||||
return databaseActiveWorkflowsList.find(
|
||||
(workflow) => workflow.id === searchParams.where.id.toString(),
|
||||
);
|
||||
}),
|
||||
update: jest.fn(),
|
||||
createQueryBuilder: jest.fn(() => {
|
||||
const fakeQueryBuilder = {
|
||||
update: () => fakeQueryBuilder,
|
||||
set: () => fakeQueryBuilder,
|
||||
where: () => fakeQueryBuilder,
|
||||
execute: async () => {},
|
||||
};
|
||||
return fakeQueryBuilder;
|
||||
}),
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const workflowCheckIfCanBeActivated = jest.fn(() => true);
|
||||
|
||||
jest
|
||||
.spyOn(Workflow.prototype, 'checkIfWorkflowCanBeActivated')
|
||||
.mockImplementation(workflowCheckIfCanBeActivated);
|
||||
|
||||
const removeFunction = jest.spyOn(ActiveWorkflowRunner.prototype, 'remove');
|
||||
const removeWebhooksFunction = jest.spyOn(ActiveWorkflowRunner.prototype, 'removeWorkflowWebhooks');
|
||||
const workflowRunnerRun = jest.spyOn(WorkflowRunner.prototype, 'run');
|
||||
const workflowExecuteAdditionalDataExecuteErrorWorkflowSpy = jest.spyOn(
|
||||
WorkflowExecuteAdditionalData,
|
||||
'executeErrorWorkflow',
|
||||
);
|
||||
|
||||
describe('ActiveWorkflowRunner', () => {
|
||||
mockInstance(ActiveExecutions);
|
||||
const externalHooks = mockInstance(ExternalHooks);
|
||||
const webhookService = mockInstance(WebhookService);
|
||||
mockInstance(Push);
|
||||
mockInstance(SecretsHelper);
|
||||
const variablesService = mockInstance(VariablesService);
|
||||
const nodesAndCredentials = mockInstance(LoadNodesAndCredentials);
|
||||
Object.assign(nodesAndCredentials, {
|
||||
loadedNodes: MOCK_NODE_TYPES_DATA,
|
||||
known: { nodes: {}, credentials: {} },
|
||||
types: { nodes: [], credentials: [] },
|
||||
});
|
||||
|
||||
const activeWorkflowRunner = Container.get(ActiveWorkflowRunner);
|
||||
|
||||
beforeAll(async () => {
|
||||
variablesService.getAllCached.mockResolvedValue([]);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await activeWorkflowRunner.removeAll();
|
||||
databaseActiveWorkflowsCount = 0;
|
||||
databaseActiveWorkflowsList = [];
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
test('Should initialize activeWorkflowRunner with empty list of active workflows and call External Hooks', async () => {
|
||||
await activeWorkflowRunner.init();
|
||||
expect(await activeWorkflowRunner.getActiveWorkflows()).toHaveLength(0);
|
||||
expect(mocked(Db.collections.Workflow.find)).toHaveBeenCalled();
|
||||
expect(externalHooks.run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('Should initialize activeWorkflowRunner with one active workflow', async () => {
|
||||
databaseActiveWorkflowsCount = 1;
|
||||
await activeWorkflowRunner.init();
|
||||
expect(await activeWorkflowRunner.getActiveWorkflows()).toHaveLength(
|
||||
databaseActiveWorkflowsCount,
|
||||
);
|
||||
expect(mocked(Db.collections.Workflow.find)).toHaveBeenCalled();
|
||||
expect(externalHooks.run).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('Should make sure function checkIfWorkflowCanBeActivated was called for every workflow', async () => {
|
||||
databaseActiveWorkflowsCount = 2;
|
||||
await activeWorkflowRunner.init();
|
||||
expect(workflowCheckIfCanBeActivated).toHaveBeenCalledTimes(databaseActiveWorkflowsCount);
|
||||
});
|
||||
|
||||
test('Call to removeAll should remove every workflow', async () => {
|
||||
databaseActiveWorkflowsCount = 2;
|
||||
await activeWorkflowRunner.init();
|
||||
expect(await activeWorkflowRunner.getActiveWorkflows()).toHaveLength(
|
||||
databaseActiveWorkflowsCount,
|
||||
);
|
||||
await activeWorkflowRunner.removeAll();
|
||||
expect(removeFunction).toHaveBeenCalledTimes(databaseActiveWorkflowsCount);
|
||||
});
|
||||
|
||||
test('Call to remove should also call removeWorkflowWebhooks', async () => {
|
||||
databaseActiveWorkflowsCount = 1;
|
||||
await activeWorkflowRunner.init();
|
||||
expect(await activeWorkflowRunner.getActiveWorkflows()).toHaveLength(
|
||||
databaseActiveWorkflowsCount,
|
||||
);
|
||||
await activeWorkflowRunner.remove('1');
|
||||
expect(removeWebhooksFunction).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('Call to isActive should return true for valid workflow', async () => {
|
||||
databaseActiveWorkflowsCount = 1;
|
||||
await activeWorkflowRunner.init();
|
||||
expect(await activeWorkflowRunner.isActive('1')).toBe(true);
|
||||
});
|
||||
|
||||
test('Call to isActive should return false for invalid workflow', async () => {
|
||||
databaseActiveWorkflowsCount = 1;
|
||||
await activeWorkflowRunner.init();
|
||||
expect(await activeWorkflowRunner.isActive('2')).toBe(false);
|
||||
});
|
||||
|
||||
test('Calling add should call checkIfWorkflowCanBeActivated', async () => {
|
||||
// Initialize with default (0) workflows
|
||||
await activeWorkflowRunner.init();
|
||||
generateWorkflows(1);
|
||||
await activeWorkflowRunner.add('1', 'activate');
|
||||
expect(workflowCheckIfCanBeActivated).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('runWorkflow should call run method in WorkflowRunner', async () => {
|
||||
await activeWorkflowRunner.init();
|
||||
const workflow = generateWorkflows(1);
|
||||
const additionalData = await WorkflowExecuteAdditionalData.getBase('fake-user-id');
|
||||
|
||||
workflowRunnerRun.mockResolvedValueOnce('invalid-execution-id');
|
||||
|
||||
await activeWorkflowRunner.runWorkflow(
|
||||
workflow[0],
|
||||
workflow[0].nodes[0],
|
||||
[[]],
|
||||
additionalData,
|
||||
'trigger',
|
||||
);
|
||||
|
||||
expect(workflowRunnerRun).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('executeErrorWorkflow should call function with same name in WorkflowExecuteAdditionalData', async () => {
|
||||
const workflowData = generateWorkflows(1)[0];
|
||||
const error = new NodeOperationError(workflowData.nodes[0], 'Fake error message');
|
||||
await activeWorkflowRunner.init();
|
||||
activeWorkflowRunner.executeErrorWorkflow(error, workflowData, 'trigger');
|
||||
expect(workflowExecuteAdditionalDataExecuteErrorWorkflowSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
describe('init()', () => {
|
||||
it('should execute error workflow on failure to activate due to 401', async () => {
|
||||
databaseActiveWorkflowsCount = 1;
|
||||
|
||||
jest.spyOn(ActiveWorkflowRunner.prototype, 'add').mockImplementation(() => {
|
||||
throw new NodeApiError(
|
||||
{
|
||||
id: 'a75dcd1b-9fed-4643-90bd-75933d67936c',
|
||||
name: 'Github Trigger',
|
||||
type: 'n8n-nodes-base.githubTrigger',
|
||||
typeVersion: 1,
|
||||
position: [0, 0],
|
||||
} as INode,
|
||||
{
|
||||
httpCode: '401',
|
||||
message: 'Authorization failed - please check your credentials',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
const executeSpy = jest.spyOn(ActiveWorkflowRunner.prototype, 'executeErrorWorkflow');
|
||||
|
||||
await activeWorkflowRunner.init();
|
||||
|
||||
const [error, workflow] = executeSpy.mock.calls[0];
|
||||
|
||||
expect(error.message).toContain('Authorization');
|
||||
expect(workflow.id).toBe('1');
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user