✨ Unify execution id + Queue system (#1340)
* Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * ✨ Add bull to execute workflows * 👕 Fix lint issue * ⚡ Add graceful shutdown to worker * ⚡ Add loading staticData to worker * 👕 Fix lint issue * ⚡ Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * ⚡ Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
This commit is contained in:
Omar Ajoue
GitHub
@@ -2,15 +2,21 @@ import {
|
||||
ActiveExecutions,
|
||||
CredentialsOverwrites,
|
||||
CredentialTypes,
|
||||
Db,
|
||||
ExternalHooks,
|
||||
IBullJobData,
|
||||
IBullJobResponse,
|
||||
ICredentialsOverwrite,
|
||||
ICredentialsTypeData,
|
||||
IExecutionFlattedDb,
|
||||
IExecutionResponse,
|
||||
IProcessMessageDataHook,
|
||||
ITransferNodeTypes,
|
||||
IWorkflowExecutionDataProcess,
|
||||
IWorkflowExecutionDataProcessWithExecution,
|
||||
NodeTypes,
|
||||
Push,
|
||||
ResponseHelper,
|
||||
WorkflowExecuteAdditionalData,
|
||||
WorkflowHelpers,
|
||||
} from './';
|
||||
@@ -21,6 +27,7 @@ import {
|
||||
} from 'n8n-core';
|
||||
|
||||
import {
|
||||
IDataObject,
|
||||
IExecutionError,
|
||||
IRun,
|
||||
Workflow,
|
||||
@@ -33,17 +40,29 @@ import * as PCancelable from 'p-cancelable';
|
||||
import { join as pathJoin } from 'path';
|
||||
import { fork } from 'child_process';
|
||||
|
||||
import * as Bull from 'bull';
|
||||
|
||||
export class WorkflowRunner {
|
||||
activeExecutions: ActiveExecutions.ActiveExecutions;
|
||||
credentialsOverwrites: ICredentialsOverwrite;
|
||||
push: Push.Push;
|
||||
jobQueue: Bull.Queue;
|
||||
|
||||
|
||||
constructor() {
|
||||
this.push = Push.getInstance();
|
||||
this.activeExecutions = ActiveExecutions.getInstance();
|
||||
this.credentialsOverwrites = CredentialsOverwrites().getAll();
|
||||
|
||||
const executionsMode = config.get('executions.mode') as string;
|
||||
|
||||
if (executionsMode === 'queue') {
|
||||
// Connect to bull-queue
|
||||
const prefix = config.get('queue.bull.prefix') as string;
|
||||
const redisOptions = config.get('queue.bull.redis') as object;
|
||||
// @ts-ignore
|
||||
this.jobQueue = new Bull('jobs', { prefix, redis: redisOptions, enableReadyCheck: false });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -99,11 +118,16 @@ export class WorkflowRunner {
|
||||
* @returns {Promise<string>}
|
||||
* @memberof WorkflowRunner
|
||||
*/
|
||||
async run(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean): Promise<string> {
|
||||
async run(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean, realtime?: boolean): Promise<string> {
|
||||
const executionsProcess = config.get('executions.process') as string;
|
||||
const executionsMode = config.get('executions.mode') as string;
|
||||
|
||||
let executionId: string;
|
||||
if (executionsProcess === 'main') {
|
||||
if (executionsMode === 'queue' && data.executionMode !== 'manual') {
|
||||
// Do not run "manual" executions in bull because sending events to the
|
||||
// frontend would not be possible
|
||||
executionId = await this.runBull(data, loadStaticData, realtime);
|
||||
} else if (executionsProcess === 'main') {
|
||||
executionId = await this.runMainProcess(data, loadStaticData);
|
||||
} else {
|
||||
executionId = await this.runSubprocess(data, loadStaticData);
|
||||
@@ -144,7 +168,7 @@ export class WorkflowRunner {
|
||||
const additionalData = await WorkflowExecuteAdditionalData.getBase(data.credentials);
|
||||
|
||||
// Register the active execution
|
||||
const executionId = this.activeExecutions.add(data, undefined);
|
||||
const executionId = await this.activeExecutions.add(data, undefined);
|
||||
|
||||
additionalData.hooks = WorkflowExecuteAdditionalData.getWorkflowHooksMain(data, executionId, true);
|
||||
|
||||
@@ -154,7 +178,7 @@ export class WorkflowRunner {
|
||||
workflowExecution = workflowExecute.processRunExecutionData(workflow);
|
||||
} else if (data.runData === undefined || data.startNodes === undefined || data.startNodes.length === 0 || data.destinationNode === undefined) {
|
||||
// Execute all nodes
|
||||
|
||||
|
||||
// Can execute without webhook so go on
|
||||
const workflowExecute = new WorkflowExecute(additionalData, data.executionMode);
|
||||
workflowExecution = workflowExecute.run(workflow, undefined, data.destinationNode);
|
||||
@@ -191,6 +215,159 @@ export class WorkflowRunner {
|
||||
return executionId;
|
||||
}
|
||||
|
||||
async runBull(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean, realtime?: boolean): Promise<string> {
|
||||
|
||||
// TODO: If "loadStaticData" is set to true it has to load data new on worker
|
||||
|
||||
// Register the active execution
|
||||
const executionId = await this.activeExecutions.add(data, undefined);
|
||||
|
||||
const jobData: IBullJobData = {
|
||||
executionId,
|
||||
loadStaticData: !!loadStaticData,
|
||||
};
|
||||
|
||||
let priority = 100;
|
||||
if (realtime === true) {
|
||||
// Jobs which require a direct response get a higher priority
|
||||
priority = 50;
|
||||
}
|
||||
// TODO: For realtime jobs should probably also not do retry or not retry if they are older than x seconds.
|
||||
// Check if they get retried by default and how often.
|
||||
const jobOptions = {
|
||||
priority,
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true,
|
||||
};
|
||||
const job = await this.jobQueue.add(jobData, jobOptions);
|
||||
console.log('Started with ID: ' + job.id.toString());
|
||||
|
||||
const hooks = WorkflowExecuteAdditionalData.getWorkflowHooksWorkerMain(data.executionMode, executionId, data.workflowData, { retryOf: data.retryOf ? data.retryOf.toString() : undefined });
|
||||
|
||||
// Normally also workflow should be supplied here but as it only used for sending
|
||||
// data to editor-UI is not needed.
|
||||
hooks.executeHookFunctions('workflowExecuteBefore', []);
|
||||
|
||||
const workflowExecution: PCancelable<IRun> = new PCancelable(async (resolve, reject, onCancel) => {
|
||||
onCancel.shouldReject = false;
|
||||
onCancel(async () => {
|
||||
if (await job.isActive()) {
|
||||
// Job is already running so tell it to stop
|
||||
await job.progress(-1);
|
||||
} else {
|
||||
// Job did not get started yet so remove from queue
|
||||
await job.remove();
|
||||
|
||||
const fullRunData: IRun = {
|
||||
data: {
|
||||
resultData: {
|
||||
error: {
|
||||
message: 'Workflow has been canceled!',
|
||||
} as IExecutionError,
|
||||
runData: {},
|
||||
},
|
||||
},
|
||||
mode: data.executionMode,
|
||||
startedAt: new Date(),
|
||||
stoppedAt: new Date(),
|
||||
};
|
||||
|
||||
this.activeExecutions.remove(executionId, fullRunData);
|
||||
resolve(fullRunData);
|
||||
}
|
||||
});
|
||||
|
||||
const jobData: Promise<IBullJobResponse> = job.finished();
|
||||
|
||||
const queueRecoveryInterval = config.get('queue.bull.queueRecoveryInterval') as number;
|
||||
|
||||
if (queueRecoveryInterval > 0) {
|
||||
/*************************************************
|
||||
* Long explanation about what this solves: *
|
||||
* This only happens in a very specific scenario *
|
||||
* when Redis crashes and recovers shortly *
|
||||
* but during this time, some execution(s) *
|
||||
* finished. The end result is that the main *
|
||||
* process will wait indefinitively and never *
|
||||
* get a response. This adds an active polling to*
|
||||
* the queue that allows us to identify that the *
|
||||
* execution finished and get information from *
|
||||
* the database. *
|
||||
*************************************************/
|
||||
let watchDogInterval: NodeJS.Timeout | undefined;
|
||||
let resolved = false;
|
||||
|
||||
const watchDog = new Promise((res) => {
|
||||
watchDogInterval = setInterval(async () => {
|
||||
const currentJob = await this.jobQueue.getJob(job.id);
|
||||
// When null means job is finished (not found in queue)
|
||||
if (currentJob === null) {
|
||||
// Mimic worker's success message
|
||||
res({success: true});
|
||||
}
|
||||
}, queueRecoveryInterval * 1000);
|
||||
});
|
||||
|
||||
|
||||
const clearWatchdogInterval = () => {
|
||||
if (watchDogInterval) {
|
||||
clearInterval(watchDogInterval);
|
||||
watchDogInterval = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
await new Promise((res, rej) => {
|
||||
jobData.then((data) => {
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
clearWatchdogInterval();
|
||||
res(data);
|
||||
}
|
||||
}).catch((e) => {
|
||||
if(!resolved) {
|
||||
resolved = true;
|
||||
clearWatchdogInterval();
|
||||
rej(e);
|
||||
}
|
||||
});
|
||||
watchDog.then((data) => {
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
clearWatchdogInterval();
|
||||
res(data);
|
||||
}
|
||||
});
|
||||
});
|
||||
} else {
|
||||
await jobData;
|
||||
}
|
||||
|
||||
|
||||
|
||||
const executionDb = await Db.collections.Execution!.findOne(executionId) as IExecutionFlattedDb;
|
||||
const fullExecutionData = ResponseHelper.unflattenExecutionData(executionDb) as IExecutionResponse;
|
||||
const runData = {
|
||||
data: fullExecutionData.data,
|
||||
finished: fullExecutionData.finished,
|
||||
mode: fullExecutionData.mode,
|
||||
startedAt: fullExecutionData.startedAt,
|
||||
stoppedAt: fullExecutionData.stoppedAt,
|
||||
} as IRun;
|
||||
|
||||
|
||||
this.activeExecutions.remove(executionId, runData);
|
||||
// Normally also static data should be supplied here but as it only used for sending
|
||||
// data to editor-UI is not needed.
|
||||
hooks.executeHookFunctions('workflowExecuteAfter', [runData]);
|
||||
|
||||
resolve(runData);
|
||||
});
|
||||
|
||||
this.activeExecutions.attachWorkflowExecution(executionId, workflowExecution);
|
||||
return executionId;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Run the workflow
|
||||
*
|
||||
@@ -209,7 +386,7 @@ export class WorkflowRunner {
|
||||
}
|
||||
|
||||
// Register the active execution
|
||||
const executionId = this.activeExecutions.add(data, subprocess);
|
||||
const executionId = await this.activeExecutions.add(data, subprocess);
|
||||
|
||||
// Check if workflow contains a "executeWorkflow" Node as in this
|
||||
// case we can not know which nodeTypes and credentialTypes will
|
||||
|
||||
Reference in New Issue
Block a user