feat(core): Add execution runData recovery and status field (#5112)
* adds ExecutionEvents view modal to ExecutionList * fix time rendering and remove wf column * checks for unfinished executions and fails them * prevent re-setting stoppedAt for execution * some cleanup / manually create rundata after crash * quicksave * remove Threads lib, log worker rewrite * cleanup comment * fix sentry destination return value * test for tests... * run tests with single worker * fix tests * remove console log * add endpoint for execution data recovery * lint cleanup and some refactoring * fix accidental recursion * remove cyclic imports * add rundata recovery to Workflowrunner * remove comments * cleanup and refactor * adds a status field to executions * setExecutionStatus on queued worker * fix onWorkflowPostExecute * set waiting from worker * get crashed status into frontend * remove comment * merge fix * cleanup * catch empty rundata in recovery * refactor IExecutionsSummary and inject nodeExecution Errors * reduce default event log size to 10mb from 100mb * add per node execution status * lint fix * merge and lint fix * phrasing change * improve preview rendering and messaging * remove debug * Improve partial rundata recovery * fix labels * fix line through * send manual rundata to ui at crash * some type and msg push fixes * improve recovered item rendering in preview * update workflowStatistics on recover * merge fix * review fixes * merge fix * notify eventbus when ui is back up * add a small timeout to make sure the UI is back up * increase reconnect timeout to 30s * adjust recover timeout and ui connection lost msg * do not stop execution in editor after x reconnects * add executionRecovered push event * fix recovered connection not green * remove reconnect toast and merge existing rundata * merge editor and recovered data for own mode
This commit is contained in:
Michael Auerswald
committed by
GitHub
GitHub
parent
3a9c257f55
commit
d143f3f2ec
@@ -1,5 +1,5 @@
|
||||
import type { MessageEventBusDestinationOptions } from 'n8n-workflow';
|
||||
import { LoggerProxy } from 'n8n-workflow';
|
||||
import type { MessageEventBusDestinationOptions } from 'n8n-workflow';
|
||||
import type { DeleteResult } from 'typeorm';
|
||||
import type { EventMessageTypes } from '../EventMessageClasses/';
|
||||
import type { MessageEventBusDestination } from '../MessageEventBusDestination/MessageEventBusDestination.ee';
|
||||
@@ -24,10 +24,16 @@ import {
|
||||
EventMessageGeneric,
|
||||
eventMessageGenericDestinationTestEvent,
|
||||
} from '../EventMessageClasses/EventMessageGeneric';
|
||||
import { recoverExecutionDataFromEventLogMessages } from './recoverEvents';
|
||||
|
||||
export type EventMessageReturnMode = 'sent' | 'unsent' | 'all' | 'unfinished';
|
||||
|
||||
class MessageEventBus extends EventEmitter {
|
||||
export interface MessageWithCallback {
|
||||
msg: EventMessageTypes;
|
||||
confirmCallback: (message: EventMessageTypes, src: EventMessageConfirmSource) => void;
|
||||
}
|
||||
|
||||
export class MessageEventBus extends EventEmitter {
|
||||
private static instance: MessageEventBus;
|
||||
|
||||
isInitialized: boolean;
|
||||
@@ -71,12 +77,13 @@ class MessageEventBus extends EventEmitter {
|
||||
if (savedEventDestinations.length > 0) {
|
||||
for (const destinationData of savedEventDestinations) {
|
||||
try {
|
||||
const destination = messageEventBusDestinationFromDb(destinationData);
|
||||
const destination = messageEventBusDestinationFromDb(this, destinationData);
|
||||
if (destination) {
|
||||
await this.addDestination(destination);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
if (error.message) LoggerProxy.debug(error.message as string);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -96,9 +103,13 @@ class MessageEventBus extends EventEmitter {
|
||||
this.logWriter?.startLogging();
|
||||
await this.send(unsentAndUnfinished.unsentMessages);
|
||||
|
||||
if (unsentAndUnfinished.unfinishedExecutions.size > 0) {
|
||||
for (const executionId of unsentAndUnfinished.unfinishedExecutions) {
|
||||
LoggerProxy.debug(`Found unfinished execution ${executionId} in event log(s)`);
|
||||
if (Object.keys(unsentAndUnfinished.unfinishedExecutions).length > 0) {
|
||||
for (const executionId of Object.keys(unsentAndUnfinished.unfinishedExecutions)) {
|
||||
await recoverExecutionDataFromEventLogMessages(
|
||||
executionId,
|
||||
unsentAndUnfinished.unfinishedExecutions[executionId],
|
||||
true,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,12 +192,15 @@ class MessageEventBus extends EventEmitter {
|
||||
}
|
||||
|
||||
async testDestination(destinationId: string): Promise<boolean> {
|
||||
const testMessage = new EventMessageGeneric({
|
||||
const msg = new EventMessageGeneric({
|
||||
eventName: eventMessageGenericDestinationTestEvent,
|
||||
});
|
||||
const destination = await this.findDestination(destinationId);
|
||||
if (destination.length > 0) {
|
||||
const sendResult = await this.destinations[destinationId].receiveFromEventBus(testMessage);
|
||||
const sendResult = await this.destinations[destinationId].receiveFromEventBus({
|
||||
msg,
|
||||
confirmCallback: () => this.confirmSent(msg, { id: '0', name: 'eventBus' }),
|
||||
});
|
||||
return sendResult;
|
||||
}
|
||||
return false;
|
||||
@@ -212,17 +226,21 @@ class MessageEventBus extends EventEmitter {
|
||||
|
||||
// generic emit for external modules to capture events
|
||||
// this is for internal use ONLY and not for use with custom destinations!
|
||||
this.emit('message', msg);
|
||||
|
||||
// LoggerProxy.debug(`Listeners: ${this.eventNames().join(',')}`);
|
||||
this.emitMessageWithCallback('message', msg);
|
||||
|
||||
if (this.shouldSendMsg(msg)) {
|
||||
for (const destinationName of Object.keys(this.destinations)) {
|
||||
this.emit(this.destinations[destinationName].getId(), msg);
|
||||
this.emitMessageWithCallback(this.destinations[destinationName].getId(), msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private emitMessageWithCallback(eventName: string, msg: EventMessageTypes): boolean {
|
||||
const confirmCallback = (message: EventMessageTypes, src: EventMessageConfirmSource) =>
|
||||
this.confirmSent(message, src);
|
||||
return this.emit(eventName, msg, confirmCallback);
|
||||
}
|
||||
|
||||
shouldSendMsg(msg: EventMessageTypes): boolean {
|
||||
return (
|
||||
isLogStreamingEnabled() &&
|
||||
@@ -249,14 +267,14 @@ class MessageEventBus extends EventEmitter {
|
||||
return filtered;
|
||||
}
|
||||
|
||||
async getUnfinishedExecutions(): Promise<Set<string>> {
|
||||
async getUnfinishedExecutions(): Promise<Record<string, EventMessageTypes[]>> {
|
||||
const queryResult = await this.logWriter?.getUnfinishedExecutions();
|
||||
return queryResult;
|
||||
}
|
||||
|
||||
async getUnsentAndUnfinishedExecutions(): Promise<{
|
||||
unsentMessages: EventMessageTypes[];
|
||||
unfinishedExecutions: Set<string>;
|
||||
unfinishedExecutions: Record<string, EventMessageTypes[]>;
|
||||
}> {
|
||||
const queryResult = await this.logWriter?.getUnsentAndUnfinishedExecutions();
|
||||
return queryResult;
|
||||
|
||||
190
packages/cli/src/eventbus/MessageEventBus/recoverEvents.ts
Normal file
190
packages/cli/src/eventbus/MessageEventBus/recoverEvents.ts
Normal file
@@ -0,0 +1,190 @@
|
||||
import { parse, stringify } from 'flatted';
|
||||
import type { IRun, IRunExecutionData, ITaskData } from 'n8n-workflow';
|
||||
import { NodeOperationError, WorkflowOperationError } from 'n8n-workflow';
|
||||
import * as Db from '@/Db';
|
||||
import type { EventMessageTypes, EventNamesTypes } from '../EventMessageClasses';
|
||||
import type { DateTime } from 'luxon';
|
||||
import { InternalHooksManager } from '../../InternalHooksManager';
|
||||
import { getPushInstance } from '@/push';
|
||||
import type { IPushDataExecutionRecovered } from '../../Interfaces';
|
||||
import { workflowExecutionCompleted } from '../../events/WorkflowStatistics';
|
||||
import { eventBus } from './MessageEventBus';
|
||||
|
||||
export async function recoverExecutionDataFromEventLogMessages(
|
||||
executionId: string,
|
||||
messages: EventMessageTypes[],
|
||||
applyToDb = true,
|
||||
): Promise<IRunExecutionData | undefined> {
|
||||
const executionEntry = await Db.collections.Execution.findOne({
|
||||
where: {
|
||||
id: executionId,
|
||||
},
|
||||
});
|
||||
|
||||
if (executionEntry && messages) {
|
||||
let executionData: IRunExecutionData | undefined;
|
||||
let workflowError: WorkflowOperationError | undefined;
|
||||
try {
|
||||
executionData = parse(executionEntry.data) as IRunExecutionData;
|
||||
} catch {}
|
||||
if (!executionData) {
|
||||
executionData = { resultData: { runData: {} } };
|
||||
}
|
||||
let nodeNames: string[] = [];
|
||||
if (
|
||||
executionData?.resultData?.runData &&
|
||||
Object.keys(executionData.resultData.runData).length > 0
|
||||
) {
|
||||
} else {
|
||||
if (!executionData.resultData) {
|
||||
executionData.resultData = {
|
||||
runData: {},
|
||||
};
|
||||
} else {
|
||||
if (!executionData.resultData.runData) {
|
||||
executionData.resultData.runData = {};
|
||||
}
|
||||
}
|
||||
}
|
||||
nodeNames = executionEntry.workflowData.nodes.map((n) => n.name);
|
||||
|
||||
let lastNodeRunTimestamp: DateTime | undefined = undefined;
|
||||
|
||||
for (const nodeName of nodeNames) {
|
||||
const nodeByName = executionEntry?.workflowData.nodes.find((n) => n.name === nodeName);
|
||||
|
||||
if (!nodeByName) continue;
|
||||
|
||||
const nodeStartedMessage = messages.find(
|
||||
(message) =>
|
||||
message.eventName === 'n8n.node.started' && message.payload.nodeName === nodeName,
|
||||
);
|
||||
const nodeFinishedMessage = messages.find(
|
||||
(message) =>
|
||||
message.eventName === 'n8n.node.finished' && message.payload.nodeName === nodeName,
|
||||
);
|
||||
|
||||
const executionTime =
|
||||
nodeStartedMessage && nodeFinishedMessage
|
||||
? nodeFinishedMessage.ts.diff(nodeStartedMessage.ts).toMillis()
|
||||
: 0;
|
||||
|
||||
let taskData: ITaskData;
|
||||
if (executionData.resultData.runData[nodeName]?.length > 0) {
|
||||
taskData = executionData.resultData.runData[nodeName][0];
|
||||
} else {
|
||||
taskData = {
|
||||
startTime: nodeStartedMessage ? nodeStartedMessage.ts.toUnixInteger() : 0,
|
||||
executionTime,
|
||||
source: [null],
|
||||
executionStatus: 'unknown',
|
||||
};
|
||||
}
|
||||
|
||||
if (nodeStartedMessage && !nodeFinishedMessage) {
|
||||
const nodeError = new NodeOperationError(
|
||||
nodeByName,
|
||||
'Node crashed, possible out-of-memory issue',
|
||||
{
|
||||
message: 'Execution stopped at this node',
|
||||
description:
|
||||
"n8n may have run out of memory while executing it. More context and tips on how to avoid this <a href='https://docs.n8n.io/flow-logic/error-handling/memory-errors' target='_blank'>in the docs</a>",
|
||||
},
|
||||
);
|
||||
workflowError = new WorkflowOperationError(
|
||||
'Workflow did not finish, possible out-of-memory issue',
|
||||
);
|
||||
taskData.error = nodeError;
|
||||
taskData.executionStatus = 'crashed';
|
||||
executionData.resultData.lastNodeExecuted = nodeName;
|
||||
if (nodeStartedMessage) lastNodeRunTimestamp = nodeStartedMessage.ts;
|
||||
} else if (nodeStartedMessage && nodeFinishedMessage) {
|
||||
taskData.executionStatus = 'success';
|
||||
if (taskData.data === undefined) {
|
||||
taskData.data = {
|
||||
main: [
|
||||
[
|
||||
{
|
||||
json: {
|
||||
isArtificalRecoveredEventItem: true,
|
||||
},
|
||||
pairedItem: undefined,
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (!executionData.resultData.runData[nodeName]) {
|
||||
executionData.resultData.runData[nodeName] = [taskData];
|
||||
}
|
||||
}
|
||||
|
||||
if (!executionData.resultData.error && workflowError) {
|
||||
executionData.resultData.error = workflowError;
|
||||
}
|
||||
if (!lastNodeRunTimestamp) {
|
||||
const workflowEndedMessage = messages.find((message) =>
|
||||
(
|
||||
[
|
||||
'n8n.workflow.success',
|
||||
'n8n.workflow.crashed',
|
||||
'n8n.workflow.failed',
|
||||
] as EventNamesTypes[]
|
||||
).includes(message.eventName),
|
||||
);
|
||||
if (workflowEndedMessage) {
|
||||
lastNodeRunTimestamp = workflowEndedMessage.ts;
|
||||
} else {
|
||||
const workflowStartedMessage = messages.find(
|
||||
(message) => message.eventName === 'n8n.workflow.started',
|
||||
);
|
||||
if (workflowStartedMessage) {
|
||||
lastNodeRunTimestamp = workflowStartedMessage.ts;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (applyToDb) {
|
||||
await Db.collections.Execution.update(executionId, {
|
||||
data: stringify(executionData),
|
||||
status: 'crashed',
|
||||
stoppedAt: lastNodeRunTimestamp?.toJSDate(),
|
||||
});
|
||||
const internalHooks = InternalHooksManager.getInstance();
|
||||
await internalHooks.onWorkflowPostExecute(executionId, executionEntry.workflowData, {
|
||||
data: executionData,
|
||||
finished: false,
|
||||
mode: executionEntry.mode,
|
||||
waitTill: executionEntry.waitTill ?? undefined,
|
||||
startedAt: executionEntry.startedAt,
|
||||
stoppedAt: lastNodeRunTimestamp?.toJSDate(),
|
||||
status: 'crashed',
|
||||
});
|
||||
const iRunData: IRun = {
|
||||
data: executionData,
|
||||
finished: false,
|
||||
mode: executionEntry.mode,
|
||||
waitTill: executionEntry.waitTill ?? undefined,
|
||||
startedAt: executionEntry.startedAt,
|
||||
stoppedAt: lastNodeRunTimestamp?.toJSDate(),
|
||||
status: 'crashed',
|
||||
};
|
||||
|
||||
// calling workflowExecutionCompleted directly because the eventEmitter is not up yet at this point
|
||||
await workflowExecutionCompleted(executionEntry.workflowData, iRunData);
|
||||
|
||||
// wait for UI to be back up and send the execution data
|
||||
eventBus.once('editorUiConnected', function handleUiBackUp() {
|
||||
// add a small timeout to make sure the UI is back up
|
||||
setTimeout(() => {
|
||||
getPushInstance().send('executionRecovered', {
|
||||
executionId,
|
||||
} as IPushDataExecutionRecovered);
|
||||
}, 1000);
|
||||
});
|
||||
}
|
||||
return executionData;
|
||||
}
|
||||
return;
|
||||
}
|
||||
Reference in New Issue
Block a user