mohiit1502/Automata
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(core): Fix execution cancellation in scaling mode (#9841)

Browse Source
This commit is contained in:
Iván Ovejero
2024-06-28 20:05:09 +02:00
committed by GitHub
parent 10f7d4b5b9
commit e613de28ca
10 changed files with 417 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

269
packages/cli/src/executions/__tests__/execution.service.test.ts Normal file
View File

@@ -0,0 +1,269 @@
import { mock } from 'jest-mock-extended';
import { WorkflowOperationError } from 'n8n-workflow';
import config from '@/config';
import { ExecutionService } from '@/executions/execution.service';
import { AbortedExecutionRetryError } from '@/errors/aborted-execution-retry.error';
import { MissingExecutionStopError } from '@/errors/missing-execution-stop.error';
import type { ActiveExecutions } from '@/ActiveExecutions';
import type { IExecutionResponse } from '@/Interfaces';
import type { Job, Queue } from '@/Queue';
import type { WaitTracker } from '@/WaitTracker';
import type { ExecutionRepository } from '@/databases/repositories/execution.repository';
import type { ExecutionRequest } from '@/executions/execution.types';
import type { ConcurrencyControlService } from '@/concurrency/concurrency-control.service';
describe('ExecutionService', () => {
const queue = mock<Queue>();
const activeExecutions = mock<ActiveExecutions>();
const executionRepository = mock<ExecutionRepository>();
const waitTracker = mock<WaitTracker>();
const concurrencyControl = mock<ConcurrencyControlService>();
const executionService = new ExecutionService(
mock(),
queue,
activeExecutions,
executionRepository,
mock(),
mock(),
waitTracker,
mock(),
concurrencyControl,
mock(),
);
beforeEach(() => {
config.set('executions.mode', 'regular');
jest.clearAllMocks();
});
describe('retry', () => {
it('should error on retrying a execution that was aborted before starting', async () => {
/**
* Arrange
*/
executionRepository.findWithUnflattenedData.mockResolvedValue(
mock<IExecutionResponse>({ data: { executionData: undefined } }),
);
const req = mock<ExecutionRequest.Retry>();
/**
* Act
*/
const retry = executionService.retry(req, []);
/**
* Assert
*/
await expect(retry).rejects.toThrow(AbortedExecutionRetryError);
});
});
describe('stop', () => {
it('should throw when stopping a missing execution', async () => {
/**
* Arrange
*/
executionRepository.findSingleExecution.mockResolvedValue(undefined);
/**
* Act
*/
const stop = executionService.stop('inexistent-123');
/**
* Assert
*/
await expect(stop).rejects.toThrowError(MissingExecutionStopError);
});
it('should throw when stopping a not-in-progress execution', async () => {
/**
* Arrange
*/
const execution = mock<IExecutionResponse>({ id: '123', status: 'success' });
executionRepository.findSingleExecution.mockResolvedValue(execution);
/**
* Act
*/
const stop = executionService.stop(execution.id);
/**
* Assert
*/
await expect(stop).rejects.toThrowError(WorkflowOperationError);
});
describe('regular mode', () => {
it('should stop a `running` execution in regular mode', async () => {
/**
* Arrange
*/
const execution = mock<IExecutionResponse>({ id: '123', status: 'running' });
executionRepository.findSingleExecution.mockResolvedValue(execution);
concurrencyControl.has.mockReturnValue(false);
activeExecutions.has.mockReturnValue(true);
waitTracker.has.mockReturnValue(false);
executionRepository.stopDuringRun.mockResolvedValue(mock<IExecutionResponse>());
/**
* Act
*/
await executionService.stop(execution.id);
/**
* Assert
*/
expect(concurrencyControl.remove).not.toHaveBeenCalled();
expect(activeExecutions.stopExecution).toHaveBeenCalledWith(execution.id);
expect(waitTracker.stopExecution).not.toHaveBeenCalled();
expect(executionRepository.stopDuringRun).toHaveBeenCalledWith(execution);
});
it('should stop a `waiting` execution in regular mode', async () => {
/**
* Arrange
*/
const execution = mock<IExecutionResponse>({ id: '123', status: 'waiting' });
executionRepository.findSingleExecution.mockResolvedValue(execution);
concurrencyControl.has.mockReturnValue(false);
activeExecutions.has.mockReturnValue(true);
waitTracker.has.mockReturnValue(true);
executionRepository.stopDuringRun.mockResolvedValue(mock<IExecutionResponse>());
/**
* Act
*/
await executionService.stop(execution.id);
/**
* Assert
*/
expect(concurrencyControl.remove).not.toHaveBeenCalled();
expect(activeExecutions.stopExecution).toHaveBeenCalledWith(execution.id);
expect(waitTracker.stopExecution).toHaveBeenCalledWith(execution.id);
expect(executionRepository.stopDuringRun).toHaveBeenCalledWith(execution);
});
it('should stop a concurrency-controlled `new` execution in regular mode', async () => {
/**
* Arrange
*/
const execution = mock<IExecutionResponse>({ id: '123', status: 'new', mode: 'trigger' });
executionRepository.findSingleExecution.mockResolvedValue(execution);
concurrencyControl.has.mockReturnValue(true);
activeExecutions.has.mockReturnValue(false);
waitTracker.has.mockReturnValue(false);
executionRepository.stopBeforeRun.mockResolvedValue(mock<IExecutionResponse>());
/**
* Act
*/
await executionService.stop(execution.id);
/**
* Assert
*/
expect(concurrencyControl.remove).toHaveBeenCalledWith({
mode: execution.mode,
executionId: execution.id,
});
expect(activeExecutions.stopExecution).not.toHaveBeenCalled();
expect(waitTracker.stopExecution).not.toHaveBeenCalled();
expect(executionRepository.stopDuringRun).not.toHaveBeenCalled();
});
});
describe('scaling mode', () => {
describe('manual execution', () => {
it('should delegate to regular mode in scaling mode', async () => {
/**
* Arrange
*/
config.set('executions.mode', 'queue');
const execution = mock<IExecutionResponse>({
id: '123',
mode: 'manual',
status: 'running',
});
executionRepository.findSingleExecution.mockResolvedValue(execution);
concurrencyControl.has.mockReturnValue(false);
activeExecutions.has.mockReturnValue(true);
waitTracker.has.mockReturnValue(false);
executionRepository.stopDuringRun.mockResolvedValue(mock<IExecutionResponse>());
// @ts-expect-error Private method
const stopInRegularModeSpy = jest.spyOn(executionService, 'stopInRegularMode');
/**
* Act
*/
await executionService.stop(execution.id);
/**
* Assert
*/
expect(stopInRegularModeSpy).toHaveBeenCalledWith(execution);
expect(activeExecutions.stopExecution).toHaveBeenCalledWith(execution.id);
expect(executionRepository.stopDuringRun).toHaveBeenCalledWith(execution);
expect(concurrencyControl.remove).not.toHaveBeenCalled();
expect(waitTracker.stopExecution).not.toHaveBeenCalled();
expect(queue.stopJob).not.toHaveBeenCalled();
});
});
describe('production execution', () => {
it('should stop a `running` execution in scaling mode', async () => {
/**
* Arrange
*/
config.set('executions.mode', 'queue');
const execution = mock<IExecutionResponse>({ id: '123', status: 'running' });
executionRepository.findSingleExecution.mockResolvedValue(execution);
waitTracker.has.mockReturnValue(false);
queue.findRunningJobBy.mockResolvedValue(mock<Job>());
executionRepository.stopDuringRun.mockResolvedValue(mock<IExecutionResponse>());
/**
* Act
*/
await executionService.stop(execution.id);
/**
* Assert
*/
expect(waitTracker.stopExecution).not.toHaveBeenCalled();
expect(queue.findRunningJobBy).toBeCalledWith({ executionId: execution.id });
expect(queue.stopJob).toHaveBeenCalled();
expect(executionRepository.stopDuringRun).toHaveBeenCalled();
});
it('should stop a `waiting` execution in scaling mode', async () => {
/**
* Arrange
*/
config.set('executions.mode', 'queue');
const execution = mock<IExecutionResponse>({ id: '123', status: 'waiting' });
executionRepository.findSingleExecution.mockResolvedValue(execution);
waitTracker.has.mockReturnValue(true);
queue.findRunningJobBy.mockResolvedValue(mock<Job>());
executionRepository.stopDuringRun.mockResolvedValue(mock<IExecutionResponse>());
/**
* Act
*/
await executionService.stop(execution.id);
/**
* Assert
*/
expect(waitTracker.stopExecution).toHaveBeenCalledWith(execution.id);
expect(queue.findRunningJobBy).toBeCalledWith({ executionId: execution.id });
expect(queue.stopJob).toHaveBeenCalled();
expect(executionRepository.stopDuringRun).toHaveBeenCalled();
});
});
});
});
});

131
packages/cli/src/executions/execution.service.ts
View File

@@ -24,7 +24,7 @@ import type {
} from '@/Interfaces';
import { NodeTypes } from '@/NodeTypes';
import { Queue } from '@/Queue';
import type { ExecutionRequest, ExecutionSummaries } from './execution.types';
import type { ExecutionRequest, ExecutionSummaries, StopResult } from './execution.types';
import { WorkflowRunner } from '@/WorkflowRunner';
import type { IGetExecutionsQueryFilter } from '@db/repositories/execution.repository';
import { ExecutionRepository } from '@db/repositories/execution.repository';
@@ -34,7 +34,7 @@ import { InternalServerError } from '@/errors/response-errors/internal-server.er
import { NotFoundError } from '@/errors/response-errors/not-found.error';
import config from '@/config';
import { WaitTracker } from '@/WaitTracker';
import type { ExecutionEntity } from '@/databases/entities/ExecutionEntity';
import { MissingExecutionStopError } from '@/errors/missing-execution-stop.error';
import { QueuedExecutionRetryError } from '@/errors/queued-execution-retry.error';
import { ConcurrencyControlService } from '@/concurrency/concurrency-control.service';
import { AbortedExecutionRetryError } from '@/errors/aborted-execution-retry.error';
@@ -328,8 +328,6 @@ export class ExecutionService {
// new API
// ----------------------------------
private readonly isRegularMode = config.getEnv('executions.mode') === 'regular';
/**
* Find summaries of executions that satisfy a query.
*
@@ -392,59 +390,6 @@ export class ExecutionService {
};
}
/**
* Stop an active execution.
*/
async stop(executionId: string) {
const execution = await this.executionRepository.findOneBy({ id: executionId });
if (!execution) throw new NotFoundError('Execution not found');
if (execution.status === 'new') {
this.concurrencyControl.remove({ mode: execution.mode, executionId });
await this.executionRepository.cancel(executionId);
return;
}
const stopResult = await this.activeExecutions.stopExecution(execution.id);
if (stopResult) return this.toExecutionStopResult(execution);
if (this.isRegularMode) {
return await this.waitTracker.stopExecution(execution.id);
}
// queue mode
try {
return await this.waitTracker.stopExecution(execution.id);
} catch {
// @TODO: Why are we swallowing this error in queue mode?
}
const activeJobs = await this.queue.getJobs(['active', 'waiting']);
const job = activeJobs.find(({ data }) => data.executionId === execution.id);
if (job) {
await this.queue.stopJob(job);
} else {
this.logger.debug('Job to stop no longer in queue', { jobId: execution.id });
}
return this.toExecutionStopResult(execution);
}
private toExecutionStopResult(execution: ExecutionEntity) {
return {
mode: execution.mode,
startedAt: new Date(execution.startedAt),
stoppedAt: execution.stoppedAt ? new Date(execution.stoppedAt) : undefined,
finished: execution.finished,
status: execution.status,
};
}
async findAllEnqueuedExecutions() {
return await this.executionRepository.findMultipleExecutions(
{
@@ -455,4 +400,76 @@ export class ExecutionService {
{ includeData: true, unflattenData: true },
);
}
async stop(executionId: string): Promise<StopResult> {
const execution = await this.executionRepository.findSingleExecution(executionId, {
includeData: true,
unflattenData: true,
});
if (!execution) throw new MissingExecutionStopError(executionId);
this.assertStoppable(execution);
const { mode, startedAt, stoppedAt, finished, status } =
config.getEnv('executions.mode') === 'regular'
? await this.stopInRegularMode(execution)
: await this.stopInScalingMode(execution);
return {
mode,
startedAt: new Date(startedAt),
stoppedAt: stoppedAt ? new Date(stoppedAt) : undefined,
finished,
status,
};
}
private assertStoppable(execution: IExecutionResponse) {
const STOPPABLE_STATUSES: ExecutionStatus[] = ['new', 'unknown', 'waiting', 'running'];
if (!STOPPABLE_STATUSES.includes(execution.status)) {
throw new WorkflowOperationError(
`Only running or waiting executions can be stopped and ${execution.id} is currently ${execution.status}`,
);
}
}
private async stopInRegularMode(execution: IExecutionResponse) {
if (this.concurrencyControl.has(execution.id)) {
this.concurrencyControl.remove({ mode: execution.mode, executionId: execution.id });
return await this.executionRepository.stopBeforeRun(execution);
}
if (this.activeExecutions.has(execution.id)) {
await this.activeExecutions.stopExecution(execution.id);
}
if (this.waitTracker.has(execution.id)) {
await this.waitTracker.stopExecution(execution.id);
}
return await this.executionRepository.stopDuringRun(execution);
}
private async stopInScalingMode(execution: IExecutionResponse) {
if (execution.mode === 'manual') {
// manual executions in scaling mode are processed by main
return await this.stopInRegularMode(execution);
}
if (this.waitTracker.has(execution.id)) {
await this.waitTracker.stopExecution(execution.id);
}
const job = await this.queue.findRunningJobBy({ executionId: execution.id });
if (job) {
await this.queue.stopJob(job);
} else {
this.logger.debug('Job to stop not in queue', { executionId: execution.id });
}
return await this.executionRepository.stopDuringRun(execution);
}
}

10
packages/cli/src/executions/execution.types.ts
View File

@@ -1,6 +1,6 @@
import type { ExecutionEntity } from '@/databases/entities/ExecutionEntity';
import type { AuthenticatedRequest } from '@/requests';
import type { ExecutionStatus, IDataObject } from 'n8n-workflow';
import type { ExecutionStatus, IDataObject, WorkflowExecuteMode } from 'n8n-workflow';
export declare namespace ExecutionRequest {
namespace QueryParams {
@@ -101,3 +101,11 @@ export type QueueRecoverySettings = {
*/
waitMs: number;
};
export type StopResult = {
mode: WorkflowExecuteMode;
startedAt: Date;
stoppedAt?: Date;
finished: boolean;
status: ExecutionStatus;
};