feat(core): Add execution runData recovery and status field (#5112)

* adds ExecutionEvents view modal to ExecutionList

* fix time rendering and remove wf column

* checks for unfinished executions and fails them

* prevent re-setting stoppedAt for execution

* some cleanup / manually create rundata after crash

* quicksave

* remove Threads lib, log worker rewrite

* cleanup comment

* fix sentry destination return value

* test for tests...

* run tests with single worker

* fix tests

* remove console log

* add endpoint for execution data recovery

* lint cleanup and some refactoring

* fix accidental recursion

* remove cyclic imports

* add rundata recovery to Workflowrunner

* remove comments

* cleanup and refactor

* adds a status field to executions

* setExecutionStatus on queued worker

* fix onWorkflowPostExecute

* set waiting from worker

* get crashed status into frontend

* remove comment

* merge fix

* cleanup

* catch empty rundata in recovery

* refactor IExecutionsSummary and inject nodeExecution Errors

* reduce default event log size to 10mb from 100mb

* add per node execution status

* lint fix

* merge and lint fix

* phrasing change

* improve preview rendering and messaging

* remove debug

* Improve partial rundata recovery

* fix labels

* fix line through

* send manual rundata to ui at crash

* some type and msg push fixes

* improve recovered item rendering in preview

* update workflowStatistics on recover

* merge fix

* review fixes

* merge fix

* notify eventbus when ui is back up

* add a small timeout to make sure the UI is back up

* increase reconnect timeout to 30s

* adjust recover timeout and ui connection lost msg

* do not stop execution in editor after x reconnects

* add executionRecovered push event

* fix recovered connection not green

* remove reconnect toast and  merge existing rundata

* merge editor and recovered data for own mode

packages/cli/package.json

@@ -22,7 +22,9 @@
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
+    "buildAndDev": "pnpm run build && pnpm run dev",
     "dev": "concurrently -k -n \"TypeScript,Node\" -c \"yellow.bold,cyan.bold\" \"npm run watch\" \"nodemon\"",
+    "dev:worker": "concurrently -k -n \"TypeScript,Node\" -c \"yellow.bold,cyan.bold\" \"npm run watch\" \"nodemon worker\"",
     "format": "prettier --write . --ignore-path ../../.prettierignore",
     "lint": "eslint --quiet .",
     "lintfix": "eslint . --fix",

packages/cli/src/ActiveExecutions.ts

@@ -5,8 +5,13 @@
 /* eslint-disable @typescript-eslint/no-unsafe-call */
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
-import type { IDeferredPromise, IExecuteResponsePromiseData, IRun } from 'n8n-workflow';
-import { createDeferredPromise } from 'n8n-workflow';
+import type {
+	IDeferredPromise,
+	IExecuteResponsePromiseData,
+	IRun,
+	ExecutionStatus,
+} from 'n8n-workflow';
+import { createDeferredPromise, LoggerProxy } from 'n8n-workflow';
 
 import type { ChildProcess } from 'child_process';
 import { stringify } from 'flatted';
@@ -36,6 +41,7 @@ export class ActiveExecutions {
 		process?: ChildProcess,
 		executionId?: string,
 	): Promise<string> {
+		let executionStatus: ExecutionStatus = executionId ? 'running' : 'new';
 		if (executionId === undefined) {
 			// Is a new execution so save in DB
 
@@ -45,6 +51,7 @@ export class ActiveExecutions {
 				finished: false,
 				startedAt: new Date(),
 				workflowData: executionData.workflowData,
+				status: executionStatus,
 			};
 
 			if (executionData.retryOf !== undefined) {
@@ -59,32 +66,37 @@ export class ActiveExecutions {
 			const execution = ResponseHelper.flattenExecutionData(fullExecutionData);
 
 			const executionResult = await Db.collections.Execution.save(execution as IExecutionFlattedDb);
+			// TODO: what is going on here?
 			executionId =
 				typeof executionResult.id === 'object'
 					? // @ts-ignore
 					  executionResult.id!.toString()
 					: executionResult.id + '';
+			if (executionId === undefined) {
+				throw new Error('There was an issue assigning an execution id to the execution');
+			}
+			executionStatus = 'running';
 		} else {
 			// Is an existing execution we want to finish so update in DB
 
-			const execution = {
+			const execution: Pick<IExecutionFlattedDb, 'id' | 'data' | 'waitTill' | 'status'> = {
 				id: executionId,
 				data: stringify(executionData.executionData!),
 				waitTill: null,
+				status: executionStatus,
 			};
 
 			await Db.collections.Execution.update(executionId, execution);
 		}
 
-		// @ts-ignore
 		this.activeExecutions[executionId] = {
 			executionData,
 			process,
 			startedAt: new Date(),
 			postExecutePromises: [],
+			status: executionStatus,
 		};
 
-		// @ts-ignore
 		return executionId;
 	}
 
@@ -215,11 +227,31 @@ export class ActiveExecutions {
 				startedAt: data.startedAt,
 				mode: data.executionData.executionMode,
 				workflowId: data.executionData.workflowData.id! as string,
+				status: data.status,
 			});
 		}
 
 		return returnData;
 	}
+
+	async setStatus(executionId: string, status: ExecutionStatus): Promise<void> {
+		if (this.activeExecutions[executionId] === undefined) {
+			LoggerProxy.debug(
+				`There is no active execution with id "${executionId}", can't update status to ${status}.`,
+			);
+			return;
+		}
+
+		this.activeExecutions[executionId].status = status;
+	}
+
+	getStatus(executionId: string): ExecutionStatus {
+		if (this.activeExecutions[executionId] === undefined) {
+			return 'unknown';
+		}
+
+		return this.activeExecutions[executionId].status;
+	}
 }
 
 let activeExecutionsInstance: ActiveExecutions | undefined;

packages/cli/src/ActiveWorkflowRunner.ts

@@ -760,6 +760,7 @@ export class ActiveWorkflowRunner {
 			mode,
 			startedAt: new Date(),
 			stoppedAt: new Date(),
+			status: 'running',
 		};
 
 		WorkflowExecuteAdditionalData.executeErrorWorkflow(workflowData, fullRunData, mode);

packages/cli/src/GenericHelpers.ts

@@ -190,6 +190,7 @@ export async function createErrorExecution(
 		workflowData,
 		workflowId: workflow.id,
 		stoppedAt: new Date(),
+		status: 'new',
 	};
 
 	const execution = ResponseHelper.flattenExecutionData(fullExecutionData);

packages/cli/src/Interfaces.ts

@@ -20,6 +20,8 @@ import type {
 	Workflow,
 	WorkflowActivateMode,
 	WorkflowExecuteMode,
+	ExecutionStatus,
+	IExecutionsSummary,
 } from 'n8n-workflow';
 
 import type { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
@@ -153,6 +155,7 @@ export interface IExecutionBase {
 	finished: boolean;
 	retryOf?: string; // If it is a retry, the id of the execution it is a retry of.
 	retrySuccessId?: string; // If it failed and a retry did succeed. The id of the successful retry.
+	status: ExecutionStatus;
 }
 
 // Data in regular format with references
@@ -188,6 +191,7 @@ export interface IExecutionFlattedDb extends IExecutionBase {
 	data: string;
 	waitTill?: Date | null;
 	workflowData: Omit<IWorkflowBase, 'pinData'>;
+	status: ExecutionStatus;
 }
 
 export interface IExecutionFlattedResponse extends IExecutionFlatted {
@@ -222,25 +226,13 @@ export interface IExecutionsStopData {
 	stoppedAt?: Date;
 }
 
-export interface IExecutionsSummary {
-	id: string;
-	finished?: boolean;
-	mode: WorkflowExecuteMode;
-	retryOf?: string;
-	retrySuccessId?: string;
-	waitTill?: Date;
-	startedAt: Date;
-	stoppedAt?: Date;
-	workflowId: string;
-	workflowName?: string;
-}
-
 export interface IExecutionsCurrentSummary {
 	id: string;
 	retryOf?: string;
 	startedAt: Date;
 	mode: WorkflowExecuteMode;
 	workflowId: string;
+	status?: ExecutionStatus;
 }
 
 export interface IExecutionDeleteFilter {
@@ -256,6 +248,7 @@ export interface IExecutingWorkflowData {
 	postExecutePromises: Array<IDeferredPromise<IRun | undefined>>;
 	responsePromise?: IDeferredPromise<IExecuteResponsePromiseData>;
 	workflowExecution?: PCancelable<IRun>;
+	status: ExecutionStatus;
 }
 
 export interface IExternalHooks {
@@ -568,7 +561,13 @@ export type IPushData =
 	| PushDataReloadNodeType
 	| PushDataRemoveNodeType
 	| PushDataTestWebhook
-	| PushDataNodeDescriptionUpdated;
+	| PushDataNodeDescriptionUpdated
+	| PushDataExecutionRecovered;
+
+type PushDataExecutionRecovered = {
+	data: IPushDataExecutionRecovered;
+	type: 'executionRecovered';
+};
 
 type PushDataExecutionFinished = {
 	data: IPushDataExecutionFinished;
@@ -615,6 +614,10 @@ type PushDataNodeDescriptionUpdated = {
 	type: 'nodeDescriptionUpdated';
 };
 
+export interface IPushDataExecutionRecovered {
+	executionId: string;
+}
+
 export interface IPushDataExecutionFinished {
 	data: IRun;
 	executionId: string;

packages/cli/src/InternalHooks.ts

@@ -4,11 +4,13 @@
 import { snakeCase } from 'change-case';
 import { BinaryDataManager } from 'n8n-core';
 import type {
+	ExecutionStatus,
 	INodesGraphResult,
 	INodeTypes,
 	IRun,
 	ITelemetryTrackProperties,
 	IWorkflowBase,
+	WorkflowExecuteMode,
 } from 'n8n-workflow';
 import { TelemetryHelpers } from 'n8n-workflow';
 import { get as pslGet } from 'psl';
@@ -26,6 +28,7 @@ import { RoleService } from './role/role.service';
 import { eventBus } from './eventbus';
 import type { User } from '@db/entities/User';
 import { N8N_VERSION } from '@/constants';
+import * as Db from '@/Db';
 
 function userToPayload(user: User): {
 	userId: string;
@@ -220,6 +223,7 @@ export class InternalHooksClass implements IInternalHooksClass {
 		data: IWorkflowExecutionDataProcess,
 	): Promise<void> {
 		void Promise.all([
+			Db.collections.Execution.update(executionId, { status: 'running' }),
 			eventBus.sendWorkflowEvent({
 				eventName: 'n8n.workflow.started',
 				payload: {
@@ -233,6 +237,24 @@ export class InternalHooksClass implements IInternalHooksClass {
 		]);
 	}
 
+	async onWorkflowCrashed(
+		executionId: string,
+		executionMode: WorkflowExecuteMode,
+		workflowData?: IWorkflowBase,
+	): Promise<void> {
+		void Promise.all([
+			eventBus.sendWorkflowEvent({
+				eventName: 'n8n.workflow.crashed',
+				payload: {
+					executionId,
+					isManual: executionMode === 'manual',
+					workflowId: workflowData?.id?.toString(),
+					workflowName: workflowData?.name,
+				},
+			}),
+		]);
+	}
+
 	async onWorkflowPostExecute(
 		executionId: string,
 		workflow: IWorkflowBase,
@@ -315,6 +337,7 @@ export class InternalHooksClass implements IInternalHooksClass {
 					user_id: userId,
 					workflow_id: workflow.id,
 					status: properties.success ? 'success' : 'failed',
+					executionStatus: runData?.status ?? 'unknown',
 					error_message: properties.error_message as string,
 					error_node_type: properties.error_node_type,
 					node_graph_string: properties.node_graph_string as string,
@@ -363,6 +386,21 @@ export class InternalHooksClass implements IInternalHooksClass {
 			}
 		}
 
+		let executionStatus: ExecutionStatus;
+		if (runData?.status === 'crashed') {
+			executionStatus = 'crashed';
+		} else if (runData?.status === 'waiting' || runData?.data?.waitTill) {
+			executionStatus = 'waiting';
+		} else {
+			executionStatus = properties.success ? 'success' : 'failed';
+		}
+
+		promises.push(
+			Db.collections.Execution.update(executionId, {
+				status: executionStatus,
+			}) as unknown as Promise<void>,
+		);
+
 		promises.push(
 			properties.success
 				? eventBus.sendWorkflowEvent({

packages/cli/src/PublicApi/types.d.ts

@@ -1,5 +1,5 @@
 import type express from 'express';
-import type { IDataObject } from 'n8n-workflow';
+import type { IDataObject, ExecutionStatus } from 'n8n-workflow';
 
 import type { User } from '@db/entities/User';
 
@@ -11,8 +11,6 @@ import type * as UserManagementMailer from '@/UserManagement/email/UserManagemen
 
 import type { Risk } from '@/audit/types';
 
-export type ExecutionStatus = 'error' | 'running' | 'success' | 'waiting' | null;
-
 export type AuthlessRequest<
 	RouteParams = {},
 	ResponseBody = {},

packages/cli/src/PublicApi/v1/handlers/executions/executions.service.ts

@@ -4,7 +4,7 @@ import { In, Not, Raw, LessThan, IsNull } from 'typeorm';
 
 import * as Db from '@/Db';
 import type { IExecutionFlattedDb, IExecutionResponseApi } from '@/Interfaces';
-import type { ExecutionStatus } from '@/PublicApi/types';
+import type { ExecutionStatus } from 'n8n-workflow';
 
 function prepareExecutionData(
 	execution: IExecutionFlattedDb | null,

packages/cli/src/ResponseHelper.ts

@@ -228,6 +228,7 @@ export function flattenExecutionData(fullExecutionData: IExecutionDb): IExecutio
 		workflowId: fullExecutionData.workflowId,
 		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 		workflowData: fullExecutionData.workflowData!,
+		status: fullExecutionData.status,
 	};
 
 	if (fullExecutionData.id !== undefined) {
@@ -261,6 +262,7 @@ export function unflattenExecutionData(fullExecutionData: IExecutionFlattedDb):
 		stoppedAt: fullExecutionData.stoppedAt,
 		finished: fullExecutionData.finished ? fullExecutionData.finished : false,
 		workflowId: fullExecutionData.workflowId,
+		status: fullExecutionData.status,
 	};
 
 	return returnData;

packages/cli/src/Server.ts

@@ -21,7 +21,7 @@ import cookieParser from 'cookie-parser';
 import express from 'express';
 import type { ServeStaticOptions } from 'serve-static';
 import type { FindManyOptions } from 'typeorm';
-import { In } from 'typeorm';
+import { Not, In } from 'typeorm';
 import type { AxiosRequestConfig } from 'axios';
 import axios from 'axios';
 import type { RequestOptions } from 'oauth-1.0a';
@@ -45,6 +45,8 @@ import type {
 	ITelemetrySettings,
 	WorkflowExecuteMode,
 	ICredentialTypes,
+	ExecutionStatus,
+	IExecutionsSummary,
 } from 'n8n-workflow';
 import { LoggerProxy, jsonParse } from 'n8n-workflow';
 
@@ -109,7 +111,6 @@ import type {
 	IDiagnosticInfo,
 	IExecutionFlattedDb,
 	IExecutionsStopData,
-	IExecutionsSummary,
 	IN8nUISettings,
 } from '@/Interfaces';
 import * as ActiveExecutions from '@/ActiveExecutions';
@@ -130,7 +131,6 @@ import { WaitTracker } from '@/WaitTracker';
 import * as WebhookHelpers from '@/WebhookHelpers';
 import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData';
 import { toHttpNodeParameters } from '@/CurlConverterHelper';
-import { eventBus } from '@/eventbus';
 import { eventBusRouter } from '@/eventbus/eventBusRoutes';
 import { isLogStreamingEnabled } from '@/eventbus/MessageEventBus/MessageEventBusHelper';
 import { getLicense } from '@/License';
@@ -145,6 +145,7 @@ import { AbstractServer } from './AbstractServer';
 import { configureMetrics } from './metrics';
 import { setupBasicAuth } from './middlewares/basicAuth';
 import { setupExternalJWTAuth } from './middlewares/externalJWTAuth';
+import { eventBus } from './eventbus';
 import { isSamlEnabled } from './Saml/helpers';
 
 const exec = promisify(callbackExec);
@@ -980,10 +981,11 @@ class Server extends AbstractServer {
 						if (!currentlyRunningExecutionIds.length) return [];
 
 						const findOptions: FindManyOptions<IExecutionFlattedDb> = {
-							select: ['id', 'workflowId', 'mode', 'retryOf', 'startedAt'],
+							select: ['id', 'workflowId', 'mode', 'retryOf', 'startedAt', 'stoppedAt', 'status'],
 							order: { id: 'DESC' },
 							where: {
 								id: In(currentlyRunningExecutionIds),
+								status: Not(In(['finished', 'stopped', 'failed', 'crashed'] as ExecutionStatus[])),
 							},
 						};
 
@@ -992,10 +994,16 @@ class Server extends AbstractServer {
 						if (!sharedWorkflowIds.length) return [];
 
 						if (req.query.filter) {
-							const { workflowId } = jsonParse<any>(req.query.filter);
+							const { workflowId, status, finished } = jsonParse<any>(req.query.filter);
 							if (workflowId && sharedWorkflowIds.includes(workflowId)) {
 								Object.assign(findOptions.where!, { workflowId });
 							}
+							if (status) {
+								Object.assign(findOptions.where!, { status: In(status) });
+							}
+							if (finished) {
+								Object.assign(findOptions.where!, { finished });
+							}
 						} else {
 							Object.assign(findOptions.where!, { workflowId: In(sharedWorkflowIds) });
 						}
@@ -1011,6 +1019,8 @@ class Server extends AbstractServer {
 								mode: execution.mode,
 								retryOf: execution.retryOf !== null ? execution.retryOf : undefined,
 								startedAt: new Date(execution.startedAt),
+								status: execution.status ?? null,
+								stoppedAt: execution.stoppedAt ?? null,
 							} as IExecutionsSummary;
 						});
 					}

packages/cli/src/WaitTracker.ts

@@ -11,7 +11,7 @@ import {
 	WorkflowOperationError,
 } from 'n8n-workflow';
 import type { FindManyOptions, ObjectLiteral } from 'typeorm';
-import { LessThanOrEqual } from 'typeorm';
+import { Not, LessThanOrEqual } from 'typeorm';
 import { DateUtils } from 'typeorm/util/DateUtils';
 
 import config from '@/config';
@@ -57,6 +57,7 @@ export class WaitTrackerClass {
 			select: ['id', 'waitTill'],
 			where: {
 				waitTill: LessThanOrEqual(new Date(Date.now() + 70000)),
+				status: Not('crashed'),
 			},
 			order: {
 				waitTill: 'ASC',
@@ -127,10 +128,13 @@ export class WaitTrackerClass {
 
 		fullExecutionData.stoppedAt = new Date();
 		fullExecutionData.waitTill = undefined;
+		fullExecutionData.status = 'canceled';
 
 		await Db.collections.Execution.update(
 			executionId,
-			ResponseHelper.flattenExecutionData(fullExecutionData),
+			ResponseHelper.flattenExecutionData({
+				...fullExecutionData,
+			}),
 		);
 
 		return {

packages/cli/src/WorkflowExecuteAdditionalData.ts

@@ -33,6 +33,7 @@ import type {
 	IWorkflowHooksOptionalParameters,
 	IWorkflowSettings,
 	WorkflowExecuteMode,
+	ExecutionStatus,
 } from 'n8n-workflow';
 import {
 	ErrorReporterProxy as ErrorReporter,
@@ -335,16 +336,22 @@ function hookFunctionsPush(): IWorkflowExecuteHooks {
 				// Clone the object except the runData. That one is not supposed
 				// to be send. Because that data got send piece by piece after
 				// each node which finished executing
-				const pushRunData = {
-					...fullRunData,
-					data: {
-						...fullRunData.data,
-						resultData: {
-							...fullRunData.data.resultData,
-							runData: {},
+				// Edit: we now DO send the runData to the UI if mode=manual so that it shows the point of crashes
+				let pushRunData;
+				if (fullRunData.mode === 'manual') {
+					pushRunData = fullRunData;
+				} else {
+					pushRunData = {
+						...fullRunData,
+						data: {
+							...fullRunData.data,
+							resultData: {
+								...fullRunData.data.resultData,
+								runData: {},
+							},
 						},
-					},
-				};
+					};
+				}
 
 				// Push data to editor-ui once workflow finished
 				Logger.debug(`Save execution progress to database for execution ID ${executionId} `, {
@@ -445,6 +452,8 @@ export function hookFunctionsPreExecute(parentProcessMode?: string): IWorkflowEx
 					// Set last executed node so that it may resume on failure
 					fullExecutionData.data.resultData.lastNodeExecuted = nodeName;
 
+					fullExecutionData.status = 'running';
+
 					const flattenedExecutionData = ResponseHelper.flattenExecutionData(fullExecutionData);
 
 					await Db.collections.Execution.update(
@@ -600,6 +609,7 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks {
 						stoppedAt: fullRunData.stoppedAt,
 						workflowData: pristineWorkflowData,
 						waitTill: fullRunData.waitTill,
+						status: fullRunData.status,
 					};
 
 					if (this.retryOf !== undefined) {
@@ -711,7 +721,11 @@ function hookFunctionsSaveWorker(): IWorkflowExecuteHooks {
 						}
 					}
 
-					const workflowDidSucceed = !fullRunData.data.resultData.error;
+					const workflowHasCrashed = fullRunData.status === 'crashed';
+					const workflowDidSucceed = !fullRunData.data.resultData.error && !workflowHasCrashed;
+					let workflowStatusFinal: ExecutionStatus = workflowDidSucceed ? 'success' : 'failed';
+					if (workflowHasCrashed) workflowStatusFinal = 'crashed';
+
 					if (!workflowDidSucceed) {
 						executeErrorWorkflow(
 							this.workflowData,
@@ -730,6 +744,7 @@ function hookFunctionsSaveWorker(): IWorkflowExecuteHooks {
 						stoppedAt: fullRunData.stoppedAt,
 						workflowData: this.workflowData,
 						waitTill: fullRunData.data.waitTill,
+						status: workflowStatusFinal,
 					};
 
 					if (this.retryOf !== undefined) {
@@ -749,6 +764,11 @@ function hookFunctionsSaveWorker(): IWorkflowExecuteHooks {
 						executionData as IExecutionFlattedDb,
 					);
 
+					// For reasons(tm) the execution status is not updated correctly in the first update, so has to be written again (tbd)
+					await Db.collections.Execution.update(this.executionId, {
+						status: executionData.status,
+					});
+
 					if (fullRunData.finished === true && this.retryOf !== undefined) {
 						// If the retry was successful save the reference it on the original execution
 						await Db.collections.Execution.update(this.retryOf, {
@@ -995,6 +1015,7 @@ async function executeWorkflow(
 			mode: 'integrated',
 			startedAt: new Date(),
 			stoppedAt: new Date(),
+			status: 'error',
 		};
 		// When failing, we might not have finished the execution
 		// Therefore, database might not contain finished errors.
@@ -1006,6 +1027,7 @@ async function executeWorkflow(
 			finished: fullRunData.finished ? fullRunData.finished : false,
 			startedAt: fullRunData.startedAt,
 			stoppedAt: fullRunData.stoppedAt,
+			status: fullRunData.status,
 			workflowData,
 		};
 		if (workflowData.id) {
@@ -1048,6 +1070,19 @@ async function executeWorkflow(
 	};
 }
 
+export function setExecutionStatus(status: ExecutionStatus) {
+	if (this.executionId === undefined) {
+		Logger.debug(`Setting execution status "${status}" failed because executionId is undefined`);
+		return;
+	}
+	Logger.debug(`Setting execution status for ${this.executionId} to "${status}"`);
+	ActiveExecutions.getInstance()
+		.setStatus(this.executionId, status)
+		.catch((error) => {
+			Logger.debug(`Setting execution status "${status}" failed: ${error.message}`);
+		});
+}
+
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export function sendMessageToUI(source: string, messages: any[]) {
 	const { sessionId } = this;
@@ -1101,6 +1136,7 @@ export async function getBase(
 		currentNodeParameters,
 		executionTimeoutTimestamp,
 		userId,
+		setExecutionStatus,
 	};
 }
 

packages/cli/src/WorkflowHelpers.ts

@@ -484,6 +484,7 @@ export function generateFailedExecutionFromError(
 		mode,
 		startedAt: new Date(),
 		stoppedAt: new Date(),
+		status: 'failed',
 	};
 }
 

packages/cli/src/WorkflowRunner.ts

@@ -55,6 +55,8 @@ import { initErrorHandling } from '@/ErrorReporting';
 import { PermissionChecker } from '@/UserManagement/PermissionChecker';
 import type { Push } from '@/push';
 import { getPushInstance } from '@/push';
+import { eventBus } from './eventbus';
+import { recoverExecutionDataFromEventLogMessages } from './eventbus/MessageEventBus/recoverEvents';
 
 export class WorkflowRunner {
 	activeExecutions: ActiveExecutions.ActiveExecutions;
@@ -103,8 +105,40 @@ export class WorkflowRunner {
 			mode: executionMode,
 			startedAt,
 			stoppedAt: new Date(),
+			status: 'error',
 		};
 
+		// The following will attempt to recover runData from event logs
+		// Note that this will only work as long as the event logs actually contain the events from this workflow execution
+		// Since processError is run almost immediately after the workflow execution has failed, it is likely that the event logs
+		// does contain those messages.
+		try {
+			// Search for messages for this executionId in event logs
+			const eventLogMessages = await eventBus.getEventsByExecutionId(executionId);
+			// Attempt to recover more better runData from these messages (but don't update the execution db entry yet)
+			if (eventLogMessages.length > 0) {
+				const eventLogExecutionData = await recoverExecutionDataFromEventLogMessages(
+					executionId,
+					eventLogMessages,
+					false,
+				);
+				if (eventLogExecutionData) {
+					fullRunData.data.resultData.runData = eventLogExecutionData.resultData.runData;
+					fullRunData.status = 'crashed';
+				}
+			}
+
+			const executionFlattedData = await Db.collections.Execution.findOneBy({ id: executionId });
+
+			void InternalHooksManager.getInstance().onWorkflowCrashed(
+				executionId,
+				executionMode,
+				executionFlattedData?.workflowData,
+			);
+		} catch {
+			// Ignore errors
+		}
+
 		// Remove from active execution with empty data. That will
 		// set the execution to failed.
 		this.activeExecutions.remove(executionId, fullRunData);
@@ -287,6 +321,10 @@ export class WorkflowRunner {
 				},
 			];
 
+			additionalData.setExecutionStatus = WorkflowExecuteAdditionalData.setExecutionStatus.bind({
+				executionId,
+			});
+
 			additionalData.sendMessageToUI = WorkflowExecuteAdditionalData.sendMessageToUI.bind({
 				sessionId: data.sessionId,
 			});
@@ -354,6 +392,7 @@ export class WorkflowRunner {
 					if (workflowExecution.isCanceled) {
 						fullRunData.finished = false;
 					}
+					fullRunData.status = this.activeExecutions.getStatus(executionId);
 					this.activeExecutions.remove(executionId, fullRunData);
 				})
 				.catch((error) => {
@@ -708,7 +747,7 @@ export class WorkflowRunner {
 				}
 
 				// eslint-disable-next-line @typescript-eslint/await-thenable
-				await this.activeExecutions.remove(message.data.executionId, message.data.result);
+				this.activeExecutions.remove(message.data.executionId, message.data.result);
 			}
 		});
 
@@ -733,7 +772,7 @@ export class WorkflowRunner {
 				);
 				// Process did exit with error code, so something went wrong.
 				const executionError = new WorkflowOperationError(
-					'Workflow execution process did crash for an unknown reason!',
+					'Workflow execution process crashed for an unknown reason!',
 				);
 
 				await this.processError(
@@ -752,7 +791,7 @@ export class WorkflowRunner {
 				// Instead of pending forever as executing when it
 				// actually isn't anymore.
 				// eslint-disable-next-line @typescript-eslint/await-thenable, no-await-in-loop
-				await this.activeExecutions.remove(executionId);
+				this.activeExecutions.remove(executionId);
 			}
 
 			clearTimeout(executionTimeout);

packages/cli/src/WorkflowRunnerProcess.ts

@@ -181,6 +181,9 @@ class WorkflowRunnerProcess {
 
 		additionalData.executionId = inputData.executionId;
 
+		additionalData.setExecutionStatus = WorkflowExecuteAdditionalData.setExecutionStatus.bind({
+			executionId: inputData.executionId,
+		});
 		// eslint-disable-next-line @typescript-eslint/no-explicit-any
 		additionalData.sendMessageToUI = async (source: string, message: any) => {
 			if (workflowRunner.data!.executionMode !== 'manual') {
@@ -487,6 +490,7 @@ process.on('message', async (message: IProcessMessage) => {
 						: ('own' as WorkflowExecuteMode),
 					startedAt: workflowRunner.startedAt,
 					stoppedAt: new Date(),
+					status: 'canceled',
 				};
 
 				// eslint-disable-next-line @typescript-eslint/no-floating-promises

packages/cli/src/commands/Interfaces.d.ts

@@ -46,8 +46,6 @@ interface INodeSpecialCase {
 	keepOnlyProperties?: string[];
 }
 
-type ExecutionStatus = 'success' | 'error' | 'warning' | 'running';
-
 declare module 'json-diff' {
 	interface IDiffOptions {
 		keysOnly?: boolean;

packages/cli/src/commands/executeBatch.ts

@@ -618,10 +618,10 @@ export class ExecuteBatch extends BaseCommand {
 					const resultError = data.data.resultData.error;
 					if (resultError) {
 						// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
-						executionResult.error = resultError.hasOwnProperty('description')
-							? // @ts-ignore
-							  resultError.description
-							: resultError.message;
+						executionResult.error =
+							resultError.hasOwnProperty('description') && resultError.description !== null
+								? resultError.description
+								: resultError.message;
 						if (data.data.resultData.lastNodeExecuted !== undefined) {
 							executionResult.error += ` on node ${data.data.resultData.lastNodeExecuted}`;
 						}

packages/cli/src/commands/worker.ts

@@ -5,7 +5,7 @@ import type PCancelable from 'p-cancelable';
 import { flags } from '@oclif/command';
 import { WorkflowExecute } from 'n8n-core';
 
-import type { IExecuteResponsePromiseData, INodeTypes, IRun } from 'n8n-workflow';
+import type { ExecutionStatus, IExecuteResponsePromiseData, INodeTypes, IRun } from 'n8n-workflow';
 import { Workflow, NodeOperationError, LoggerProxy, sleep } from 'n8n-workflow';
 
 import * as Db from '@/Db';
@@ -189,6 +189,11 @@ export class Worker extends BaseCommand {
 
 		additionalData.executionId = executionId;
 
+		additionalData.setExecutionStatus = (status: ExecutionStatus) => {
+			// Can't set the status directly in the queued worker, but it will happen in InternalHook.onWorkflowPostExecute
+			LoggerProxy.debug(`Queued worker execution status for ${executionId} is "${status}"`);
+		};
+
 		let workflowExecute: WorkflowExecute;
 		let workflowRun: PCancelable<IRun>;
 		if (currentExecutionDb.data !== undefined) {

packages/cli/src/config/schema.ts

@@ -1150,7 +1150,7 @@ export const schema = {
 			maxFileSizeInKB: {
 				doc: 'Maximum size of an event log file before a new one is started.',
 				format: Number,
-				default: 102400, // 100MB
+				default: 10240, // 10MB
 				env: 'N8N_EVENTBUS_LOGWRITER_MAXFILESIZEINKB',
 			},
 			logBaseName: {

packages/cli/src/databases/entities/ExecutionEntity.ts

@@ -1,4 +1,4 @@
-import { WorkflowExecuteMode } from 'n8n-workflow';
+import { ExecutionStatus, WorkflowExecuteMode } from 'n8n-workflow';
 import { Column, Entity, Generated, Index, PrimaryColumn } from 'typeorm';
 import { datetimeColumnType, jsonColumnType } from './AbstractEntity';
 import { IWorkflowDb } from '@/Interfaces';
@@ -31,6 +31,9 @@ export class ExecutionEntity implements IExecutionFlattedDb {
 	@Column({ nullable: true })
 	retrySuccessId: string;
 
+	@Column('varchar', { nullable: true })
+	status: ExecutionStatus;
+
 	@Column(datetimeColumnType)
 	startedAt: Date;
 

packages/cli/src/databases/migrations/mysqldb/1674138566000-AddStatusToExecutions.ts

@@ -0,0 +1,24 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+import { logMigrationEnd, logMigrationStart } from '@db/utils/migrationHelpers';
+import config from '@/config';
+
+export class AddStatusToExecutions1674138566000 implements MigrationInterface {
+	name = 'AddStatusToExecutions1674138566000';
+	public async up(queryRunner: QueryRunner): Promise<void> {
+		logMigrationStart(this.name);
+		const tablePrefix = config.getEnv('database.tablePrefix');
+
+		await queryRunner.query(
+			// 'ALTER TABLE `' + tablePrefix + 'execution_entity` ADD COLUMN `status` varchar',
+			`ALTER TABLE \`${tablePrefix}execution_entity\`	ADD COLUMN \`status\` VARCHAR(255)`,
+		);
+
+		logMigrationEnd(this.name);
+	}
+
+	public async down(queryRunner: QueryRunner): Promise<void> {
+		const tablePrefix = config.getEnv('database.tablePrefix');
+
+		await queryRunner.query(`ALTER TABLE \`${tablePrefix}execution_entity\`	DROP COLUMN \`status\``);
+	}
+}

packages/cli/src/databases/migrations/mysqldb/index.ts

@@ -31,6 +31,7 @@ import { MessageEventBusDestinations1671535397530 } from './1671535397530-Messag
 import { DeleteExecutionsWithWorkflows1673268682475 } from './1673268682475-DeleteExecutionsWithWorkflows';
 import { CreateLdapEntities1674509946020 } from './1674509946020-CreateLdapEntities';
 import { PurgeInvalidWorkflowConnections1675940580449 } from './1675940580449-PurgeInvalidWorkflowConnections';
+import { AddStatusToExecutions1674138566000 } from './1674138566000-AddStatusToExecutions';
 
 export const mysqlMigrations = [
 	InitialMigration1588157391238,
@@ -65,5 +66,6 @@ export const mysqlMigrations = [
 	MessageEventBusDestinations1671535397530,
 	DeleteExecutionsWithWorkflows1673268682475,
 	CreateLdapEntities1674509946020,
-	PurgeInvalidWorkflowConnections1675940580449
+	PurgeInvalidWorkflowConnections1675940580449,
+	AddStatusToExecutions1674138566000,
 ];

packages/cli/src/databases/migrations/postgresdb/1674138566000-AddStatusToExecutions.ts

@@ -0,0 +1,21 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+import { logMigrationEnd, logMigrationStart } from '@db/utils/migrationHelpers';
+import config from '@/config';
+
+export class AddStatusToExecutions1674138566000 implements MigrationInterface {
+	name = 'AddStatusToExecutions1674138566000';
+	public async up(queryRunner: QueryRunner): Promise<void> {
+		logMigrationStart(this.name);
+		const tablePrefix = config.getEnv('database.tablePrefix');
+
+		await queryRunner.query(`ALTER TABLE ${tablePrefix}execution_entity ADD COLUMN status varchar`);
+
+		logMigrationEnd(this.name);
+	}
+
+	public async down(queryRunner: QueryRunner): Promise<void> {
+		const tablePrefix = config.getEnv('database.tablePrefix');
+
+		await queryRunner.query(`ALTER TABLE ${tablePrefix}execution_entity DROP COLUMN status`);
+	}
+}

packages/cli/src/databases/migrations/postgresdb/index.ts

@@ -29,6 +29,7 @@ import { MessageEventBusDestinations1671535397530 } from './1671535397530-Messag
 import { DeleteExecutionsWithWorkflows1673268682475 } from './1673268682475-DeleteExecutionsWithWorkflows';
 import { CreateLdapEntities1674509946020 } from './1674509946020-CreateLdapEntities';
 import { PurgeInvalidWorkflowConnections1675940580449 } from './1675940580449-PurgeInvalidWorkflowConnections';
+import { AddStatusToExecutions1674138566000 } from './1674138566000-AddStatusToExecutions';
 
 export const postgresMigrations = [
 	InitialMigration1587669153312,
@@ -61,5 +62,6 @@ export const postgresMigrations = [
 	MessageEventBusDestinations1671535397530,
 	DeleteExecutionsWithWorkflows1673268682475,
 	CreateLdapEntities1674509946020,
-	PurgeInvalidWorkflowConnections1675940580449
+	PurgeInvalidWorkflowConnections1675940580449,
+	AddStatusToExecutions1674138566000,
 ];

packages/cli/src/databases/migrations/sqlite/1674138566000-AddStatusToExecutions.ts

@@ -0,0 +1,23 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+import { logMigrationEnd, logMigrationStart } from '@db/utils/migrationHelpers';
+import config from '@/config';
+
+export class AddStatusToExecutions1674138566000 implements MigrationInterface {
+	name = 'AddStatusToExecutions1674138566000';
+	public async up(queryRunner: QueryRunner): Promise<void> {
+		logMigrationStart(this.name);
+		const tablePrefix = config.getEnv('database.tablePrefix');
+
+		await queryRunner.query(
+			`ALTER TABLE \`${tablePrefix}execution_entity\` ADD COLUMN "status" varchar`,
+		);
+
+		logMigrationEnd(this.name);
+	}
+
+	public async down(queryRunner: QueryRunner): Promise<void> {
+		const tablePrefix = config.getEnv('database.tablePrefix');
+
+		await queryRunner.query(`ALTER TABLE \`${tablePrefix}execution_entity\` DROP COLUMN "status"`);
+	}
+}

packages/cli/src/databases/migrations/sqlite/index.ts

@@ -28,6 +28,7 @@ import { MessageEventBusDestinations1671535397530 } from './1671535397530-Messag
 import { DeleteExecutionsWithWorkflows1673268682475 } from './1673268682475-DeleteExecutionsWithWorkflows';
 import { CreateLdapEntities1674509946020 } from './1674509946020-CreateLdapEntities';
 import { PurgeInvalidWorkflowConnections1675940580449 } from './1675940580449-PurgeInvalidWorkflowConnections';
+import { AddStatusToExecutions1674138566000 } from './1674138566000-AddStatusToExecutions';
 
 const sqliteMigrations = [
 	InitialMigration1588102412422,
@@ -60,6 +61,7 @@ const sqliteMigrations = [
 	DeleteExecutionsWithWorkflows1673268682475,
 	CreateLdapEntities1674509946020,
 	PurgeInvalidWorkflowConnections1675940580449,
+	AddStatusToExecutions1674138566000,
 ];
 
 export { sqliteMigrations };

packages/cli/src/eventbus/EventMessageClasses/AbstractEventMessage.ts

@@ -4,6 +4,7 @@ import type { EventMessageTypeNames, JsonObject } from 'n8n-workflow';
 import { v4 as uuid } from 'uuid';
 import type { AbstractEventPayload } from './AbstractEventPayload';
 import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
+import type { EventNamesTypes } from '.';
 
 function modifyUnderscoredKeys(
 	input: { [key: string]: any },
@@ -85,7 +86,7 @@ export abstract class AbstractEventMessage {
 
 	ts: DateTime;
 
-	eventName: string;
+	eventName: EventNamesTypes;
 
 	message: string;
 

packages/cli/src/eventbus/EventMessageClasses/AbstractEventMessageOptions.ts

@@ -1,12 +1,13 @@
 import type { DateTime } from 'luxon';
 import type { EventMessageTypeNames } from 'n8n-workflow';
+import type { EventNamesTypes } from '.';
 import type { AbstractEventPayload } from './AbstractEventPayload';
 
 export interface AbstractEventMessageOptions {
 	__type?: EventMessageTypeNames;
 	id?: string;
 	ts?: DateTime | string;
-	eventName: string;
+	eventName: EventNamesTypes;
 	message?: string;
 	payload?: AbstractEventPayload;
 	anonymize?: boolean;

packages/cli/src/eventbus/EventMessageClasses/EventMessageAudit.ts

@@ -1,31 +1,9 @@
 import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage';
-import type { JsonObject, JsonValue } from 'n8n-workflow';
 import { EventMessageTypeNames } from 'n8n-workflow';
+import type { JsonObject, JsonValue } from 'n8n-workflow';
 import type { AbstractEventPayload } from './AbstractEventPayload';
 import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
-
-export const eventNamesAudit = [
-	'n8n.audit.user.signedup',
-	'n8n.audit.user.updated',
-	'n8n.audit.user.deleted',
-	'n8n.audit.user.invited',
-	'n8n.audit.user.invitation.accepted',
-	'n8n.audit.user.reinvited',
-	'n8n.audit.user.email.failed',
-	'n8n.audit.user.reset.requested',
-	'n8n.audit.user.reset',
-	'n8n.audit.user.credentials.created',
-	'n8n.audit.user.credentials.shared',
-	'n8n.audit.user.api.created',
-	'n8n.audit.user.api.deleted',
-	'n8n.audit.package.installed',
-	'n8n.audit.package.updated',
-	'n8n.audit.package.deleted',
-	'n8n.audit.workflow.created',
-	'n8n.audit.workflow.deleted',
-	'n8n.audit.workflow.updated',
-] as const;
-export type EventNamesAuditType = (typeof eventNamesAudit)[number];
+import type { EventNamesAuditType } from '.';
 
 // --------------------------------------
 // EventMessage class for Audit events

packages/cli/src/eventbus/EventMessageClasses/EventMessageNode.ts

@@ -3,9 +3,7 @@ import type { JsonObject } from 'n8n-workflow';
 import { EventMessageTypeNames } from 'n8n-workflow';
 import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
 import type { AbstractEventPayload } from './AbstractEventPayload';
-
-export const eventNamesNode = ['n8n.node.started', 'n8n.node.finished'] as const;
-export type EventNamesNodeType = (typeof eventNamesNode)[number];
+import type { EventNamesNodeType } from '.';
 
 // --------------------------------------
 // EventMessage class for Node events

packages/cli/src/eventbus/EventMessageClasses/EventMessageWorkflow.ts

@@ -4,14 +4,7 @@ import { EventMessageTypeNames } from 'n8n-workflow';
 import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
 import type { AbstractEventPayload } from './AbstractEventPayload';
 import type { IExecutionBase } from '@/Interfaces';
-
-export const eventNamesWorkflow = [
-	'n8n.workflow.started',
-	'n8n.workflow.success',
-	'n8n.workflow.failed',
-] as const;
-
-export type EventNamesWorkflowType = (typeof eventNamesWorkflow)[number];
+import type { EventNamesWorkflowType } from '.';
 
 // --------------------------------------
 // EventMessage class for Workflow events

packages/cli/src/eventbus/EventMessageClasses/index.ts

@@ -1,12 +1,47 @@
-import type { EventMessageAudit, EventNamesAuditType } from './EventMessageAudit';
-import { eventNamesAudit } from './EventMessageAudit';
+import type { EventMessageAudit } from './EventMessageAudit';
 import type { EventMessageGeneric } from './EventMessageGeneric';
-import type { EventMessageNode, EventNamesNodeType } from './EventMessageNode';
-import { eventNamesNode } from './EventMessageNode';
-import type { EventMessageWorkflow, EventNamesWorkflowType } from './EventMessageWorkflow';
-import { eventNamesWorkflow } from './EventMessageWorkflow';
+import type { EventMessageNode } from './EventMessageNode';
+import type { EventMessageWorkflow } from './EventMessageWorkflow';
+
+export const eventNamesWorkflow = [
+	'n8n.workflow.started',
+	'n8n.workflow.success',
+	'n8n.workflow.failed',
+	'n8n.workflow.crashed',
+] as const;
+export const eventNamesNode = ['n8n.node.started', 'n8n.node.finished'] as const;
+export const eventNamesAudit = [
+	'n8n.audit.user.signedup',
+	'n8n.audit.user.updated',
+	'n8n.audit.user.deleted',
+	'n8n.audit.user.invited',
+	'n8n.audit.user.invitation.accepted',
+	'n8n.audit.user.reinvited',
+	'n8n.audit.user.email.failed',
+	'n8n.audit.user.reset.requested',
+	'n8n.audit.user.reset',
+	'n8n.audit.user.credentials.created',
+	'n8n.audit.user.credentials.shared',
+	'n8n.audit.user.api.created',
+	'n8n.audit.user.api.deleted',
+	'n8n.audit.package.installed',
+	'n8n.audit.package.updated',
+	'n8n.audit.package.deleted',
+	'n8n.audit.workflow.created',
+	'n8n.audit.workflow.deleted',
+	'n8n.audit.workflow.updated',
+] as const;
+
+export type EventNamesWorkflowType = (typeof eventNamesWorkflow)[number];
+export type EventNamesAuditType = (typeof eventNamesAudit)[number];
+export type EventNamesNodeType = (typeof eventNamesNode)[number];
+
+export type EventNamesTypes =
+	| EventNamesAuditType
+	| EventNamesWorkflowType
+	| EventNamesNodeType
+	| 'n8n.destination.test';
 
-export type EventNamesTypes = EventNamesAuditType | EventNamesWorkflowType | EventNamesNodeType;
 export const eventNamesAll = [...eventNamesAudit, ...eventNamesWorkflow, ...eventNamesNode];
 
 export type EventMessageTypes =

packages/cli/src/eventbus/MessageEventBus/MessageEventBus.ts

@@ -1,5 +1,5 @@
-import type { MessageEventBusDestinationOptions } from 'n8n-workflow';
 import { LoggerProxy } from 'n8n-workflow';
+import type { MessageEventBusDestinationOptions } from 'n8n-workflow';
 import type { DeleteResult } from 'typeorm';
 import type { EventMessageTypes } from '../EventMessageClasses/';
 import type { MessageEventBusDestination } from '../MessageEventBusDestination/MessageEventBusDestination.ee';
@@ -24,10 +24,16 @@ import {
 	EventMessageGeneric,
 	eventMessageGenericDestinationTestEvent,
 } from '../EventMessageClasses/EventMessageGeneric';
+import { recoverExecutionDataFromEventLogMessages } from './recoverEvents';
 
 export type EventMessageReturnMode = 'sent' | 'unsent' | 'all' | 'unfinished';
 
-class MessageEventBus extends EventEmitter {
+export interface MessageWithCallback {
+	msg: EventMessageTypes;
+	confirmCallback: (message: EventMessageTypes, src: EventMessageConfirmSource) => void;
+}
+
+export class MessageEventBus extends EventEmitter {
 	private static instance: MessageEventBus;
 
 	isInitialized: boolean;
@@ -71,12 +77,13 @@ class MessageEventBus extends EventEmitter {
 		if (savedEventDestinations.length > 0) {
 			for (const destinationData of savedEventDestinations) {
 				try {
-					const destination = messageEventBusDestinationFromDb(destinationData);
+					const destination = messageEventBusDestinationFromDb(this, destinationData);
 					if (destination) {
 						await this.addDestination(destination);
 					}
 				} catch (error) {
-					console.log(error);
+					// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+					if (error.message) LoggerProxy.debug(error.message as string);
 				}
 			}
 		}
@@ -96,9 +103,13 @@ class MessageEventBus extends EventEmitter {
 		this.logWriter?.startLogging();
 		await this.send(unsentAndUnfinished.unsentMessages);
 
-		if (unsentAndUnfinished.unfinishedExecutions.size > 0) {
-			for (const executionId of unsentAndUnfinished.unfinishedExecutions) {
-				LoggerProxy.debug(`Found unfinished execution ${executionId} in event log(s)`);
+		if (Object.keys(unsentAndUnfinished.unfinishedExecutions).length > 0) {
+			for (const executionId of Object.keys(unsentAndUnfinished.unfinishedExecutions)) {
+				await recoverExecutionDataFromEventLogMessages(
+					executionId,
+					unsentAndUnfinished.unfinishedExecutions[executionId],
+					true,
+				);
 			}
 		}
 
@@ -181,12 +192,15 @@ class MessageEventBus extends EventEmitter {
 	}
 
 	async testDestination(destinationId: string): Promise<boolean> {
-		const testMessage = new EventMessageGeneric({
+		const msg = new EventMessageGeneric({
 			eventName: eventMessageGenericDestinationTestEvent,
 		});
 		const destination = await this.findDestination(destinationId);
 		if (destination.length > 0) {
-			const sendResult = await this.destinations[destinationId].receiveFromEventBus(testMessage);
+			const sendResult = await this.destinations[destinationId].receiveFromEventBus({
+				msg,
+				confirmCallback: () => this.confirmSent(msg, { id: '0', name: 'eventBus' }),
+			});
 			return sendResult;
 		}
 		return false;
@@ -212,17 +226,21 @@ class MessageEventBus extends EventEmitter {
 
 		// generic emit for external modules to capture events
 		// this is for internal use ONLY and not for use with custom destinations!
-		this.emit('message', msg);
-
-		// LoggerProxy.debug(`Listeners: ${this.eventNames().join(',')}`);
+		this.emitMessageWithCallback('message', msg);
 
 		if (this.shouldSendMsg(msg)) {
 			for (const destinationName of Object.keys(this.destinations)) {
-				this.emit(this.destinations[destinationName].getId(), msg);
+				this.emitMessageWithCallback(this.destinations[destinationName].getId(), msg);
 			}
 		}
 	}
 
+	private emitMessageWithCallback(eventName: string, msg: EventMessageTypes): boolean {
+		const confirmCallback = (message: EventMessageTypes, src: EventMessageConfirmSource) =>
+			this.confirmSent(message, src);
+		return this.emit(eventName, msg, confirmCallback);
+	}
+
 	shouldSendMsg(msg: EventMessageTypes): boolean {
 		return (
 			isLogStreamingEnabled() &&
@@ -249,14 +267,14 @@ class MessageEventBus extends EventEmitter {
 		return filtered;
 	}
 
-	async getUnfinishedExecutions(): Promise<Set<string>> {
+	async getUnfinishedExecutions(): Promise<Record<string, EventMessageTypes[]>> {
 		const queryResult = await this.logWriter?.getUnfinishedExecutions();
 		return queryResult;
 	}
 
 	async getUnsentAndUnfinishedExecutions(): Promise<{
 		unsentMessages: EventMessageTypes[];
-		unfinishedExecutions: Set<string>;
+		unfinishedExecutions: Record<string, EventMessageTypes[]>;
 	}> {
 		const queryResult = await this.logWriter?.getUnsentAndUnfinishedExecutions();
 		return queryResult;

packages/cli/src/eventbus/MessageEventBus/recoverEvents.ts

@@ -0,0 +1,190 @@
+import { parse, stringify } from 'flatted';
+import type { IRun, IRunExecutionData, ITaskData } from 'n8n-workflow';
+import { NodeOperationError, WorkflowOperationError } from 'n8n-workflow';
+import * as Db from '@/Db';
+import type { EventMessageTypes, EventNamesTypes } from '../EventMessageClasses';
+import type { DateTime } from 'luxon';
+import { InternalHooksManager } from '../../InternalHooksManager';
+import { getPushInstance } from '@/push';
+import type { IPushDataExecutionRecovered } from '../../Interfaces';
+import { workflowExecutionCompleted } from '../../events/WorkflowStatistics';
+import { eventBus } from './MessageEventBus';
+
+export async function recoverExecutionDataFromEventLogMessages(
+	executionId: string,
+	messages: EventMessageTypes[],
+	applyToDb = true,
+): Promise<IRunExecutionData | undefined> {
+	const executionEntry = await Db.collections.Execution.findOne({
+		where: {
+			id: executionId,
+		},
+	});
+
+	if (executionEntry && messages) {
+		let executionData: IRunExecutionData | undefined;
+		let workflowError: WorkflowOperationError | undefined;
+		try {
+			executionData = parse(executionEntry.data) as IRunExecutionData;
+		} catch {}
+		if (!executionData) {
+			executionData = { resultData: { runData: {} } };
+		}
+		let nodeNames: string[] = [];
+		if (
+			executionData?.resultData?.runData &&
+			Object.keys(executionData.resultData.runData).length > 0
+		) {
+		} else {
+			if (!executionData.resultData) {
+				executionData.resultData = {
+					runData: {},
+				};
+			} else {
+				if (!executionData.resultData.runData) {
+					executionData.resultData.runData = {};
+				}
+			}
+		}
+		nodeNames = executionEntry.workflowData.nodes.map((n) => n.name);
+
+		let lastNodeRunTimestamp: DateTime | undefined = undefined;
+
+		for (const nodeName of nodeNames) {
+			const nodeByName = executionEntry?.workflowData.nodes.find((n) => n.name === nodeName);
+
+			if (!nodeByName) continue;
+
+			const nodeStartedMessage = messages.find(
+				(message) =>
+					message.eventName === 'n8n.node.started' && message.payload.nodeName === nodeName,
+			);
+			const nodeFinishedMessage = messages.find(
+				(message) =>
+					message.eventName === 'n8n.node.finished' && message.payload.nodeName === nodeName,
+			);
+
+			const executionTime =
+				nodeStartedMessage && nodeFinishedMessage
+					? nodeFinishedMessage.ts.diff(nodeStartedMessage.ts).toMillis()
+					: 0;
+
+			let taskData: ITaskData;
+			if (executionData.resultData.runData[nodeName]?.length > 0) {
+				taskData = executionData.resultData.runData[nodeName][0];
+			} else {
+				taskData = {
+					startTime: nodeStartedMessage ? nodeStartedMessage.ts.toUnixInteger() : 0,
+					executionTime,
+					source: [null],
+					executionStatus: 'unknown',
+				};
+			}
+
+			if (nodeStartedMessage && !nodeFinishedMessage) {
+				const nodeError = new NodeOperationError(
+					nodeByName,
+					'Node crashed, possible out-of-memory issue',
+					{
+						message: 'Execution stopped at this node',
+						description:
+							"n8n may have run out of memory while executing it. More context and tips on how to avoid this <a href='https://docs.n8n.io/flow-logic/error-handling/memory-errors' target='_blank'>in the docs</a>",
+					},
+				);
+				workflowError = new WorkflowOperationError(
+					'Workflow did not finish, possible out-of-memory issue',
+				);
+				taskData.error = nodeError;
+				taskData.executionStatus = 'crashed';
+				executionData.resultData.lastNodeExecuted = nodeName;
+				if (nodeStartedMessage) lastNodeRunTimestamp = nodeStartedMessage.ts;
+			} else if (nodeStartedMessage && nodeFinishedMessage) {
+				taskData.executionStatus = 'success';
+				if (taskData.data === undefined) {
+					taskData.data = {
+						main: [
+							[
+								{
+									json: {
+										isArtificalRecoveredEventItem: true,
+									},
+									pairedItem: undefined,
+								},
+							],
+						],
+					};
+				}
+			}
+
+			if (!executionData.resultData.runData[nodeName]) {
+				executionData.resultData.runData[nodeName] = [taskData];
+			}
+		}
+
+		if (!executionData.resultData.error && workflowError) {
+			executionData.resultData.error = workflowError;
+		}
+		if (!lastNodeRunTimestamp) {
+			const workflowEndedMessage = messages.find((message) =>
+				(
+					[
+						'n8n.workflow.success',
+						'n8n.workflow.crashed',
+						'n8n.workflow.failed',
+					] as EventNamesTypes[]
+				).includes(message.eventName),
+			);
+			if (workflowEndedMessage) {
+				lastNodeRunTimestamp = workflowEndedMessage.ts;
+			} else {
+				const workflowStartedMessage = messages.find(
+					(message) => message.eventName === 'n8n.workflow.started',
+				);
+				if (workflowStartedMessage) {
+					lastNodeRunTimestamp = workflowStartedMessage.ts;
+				}
+			}
+		}
+		if (applyToDb) {
+			await Db.collections.Execution.update(executionId, {
+				data: stringify(executionData),
+				status: 'crashed',
+				stoppedAt: lastNodeRunTimestamp?.toJSDate(),
+			});
+			const internalHooks = InternalHooksManager.getInstance();
+			await internalHooks.onWorkflowPostExecute(executionId, executionEntry.workflowData, {
+				data: executionData,
+				finished: false,
+				mode: executionEntry.mode,
+				waitTill: executionEntry.waitTill ?? undefined,
+				startedAt: executionEntry.startedAt,
+				stoppedAt: lastNodeRunTimestamp?.toJSDate(),
+				status: 'crashed',
+			});
+			const iRunData: IRun = {
+				data: executionData,
+				finished: false,
+				mode: executionEntry.mode,
+				waitTill: executionEntry.waitTill ?? undefined,
+				startedAt: executionEntry.startedAt,
+				stoppedAt: lastNodeRunTimestamp?.toJSDate(),
+				status: 'crashed',
+			};
+
+			// calling workflowExecutionCompleted directly because the eventEmitter is not up yet at this point
+			await workflowExecutionCompleted(executionEntry.workflowData, iRunData);
+
+			// wait for UI to be back up and send the execution data
+			eventBus.once('editorUiConnected', function handleUiBackUp() {
+				// add a small timeout to make sure the UI is back up
+				setTimeout(() => {
+					getPushInstance().send('executionRecovered', {
+						executionId,
+					} as IPushDataExecutionRecovered);
+				}, 1000);
+			});
+		}
+		return executionData;
+	}
+	return;
+}

packages/cli/src/eventbus/MessageEventBusDestination/Helpers.ee.ts

@@ -1,5 +1,4 @@
-/* eslint-disable import/no-cycle */
-import type { EventDestinations } from '@db/entities/MessageEventBusDestinationEntity';
+import type { EventDestinations } from '@/databases/entities/MessageEventBusDestinationEntity';
 import { promClient } from '@/metrics';
 import {
 	EventMessageTypeNames,
@@ -12,21 +11,23 @@ import type { MessageEventBusDestination } from './MessageEventBusDestination.ee
 import { MessageEventBusDestinationSentry } from './MessageEventBusDestinationSentry.ee';
 import { MessageEventBusDestinationSyslog } from './MessageEventBusDestinationSyslog.ee';
 import { MessageEventBusDestinationWebhook } from './MessageEventBusDestinationWebhook.ee';
+import type { MessageEventBus } from '../MessageEventBus/MessageEventBus';
 
 export function messageEventBusDestinationFromDb(
+	eventBusInstance: MessageEventBus,
 	dbData: EventDestinations,
 ): MessageEventBusDestination | null {
 	const destinationData = dbData.destination;
 	if ('__type' in destinationData) {
 		switch (destinationData.__type) {
 			case MessageEventBusDestinationTypeNames.sentry:
-				return MessageEventBusDestinationSentry.deserialize(destinationData);
+				return MessageEventBusDestinationSentry.deserialize(eventBusInstance, destinationData);
 			case MessageEventBusDestinationTypeNames.syslog:
-				return MessageEventBusDestinationSyslog.deserialize(destinationData);
+				return MessageEventBusDestinationSyslog.deserialize(eventBusInstance, destinationData);
 			case MessageEventBusDestinationTypeNames.webhook:
-				return MessageEventBusDestinationWebhook.deserialize(destinationData);
+				return MessageEventBusDestinationWebhook.deserialize(eventBusInstance, destinationData);
 			default:
-				console.log('MessageEventBusDestination __type unknown');
+				LoggerProxy.debug('MessageEventBusDestination __type unknown');
 		}
 	}
 	return null;

packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestination.ee.ts

@@ -4,14 +4,17 @@ import { LoggerProxy, MessageEventBusDestinationTypeNames } from 'n8n-workflow';
 import * as Db from '@/Db';
 import type { AbstractEventMessage } from '../EventMessageClasses/AbstractEventMessage';
 import type { EventMessageTypes } from '../EventMessageClasses';
-import { eventBus } from '..';
 import type { DeleteResult, InsertResult } from 'typeorm';
+import type { EventMessageConfirmSource } from '../EventMessageClasses/EventMessageConfirm';
+import type { MessageEventBus, MessageWithCallback } from '../MessageEventBus/MessageEventBus';
 
 export abstract class MessageEventBusDestination implements MessageEventBusDestinationOptions {
 	// Since you can't have static abstract functions - this just serves as a reminder that you need to implement these. Please.
 	// static abstract deserialize(): MessageEventBusDestination | null;
 	readonly id: string;
 
+	readonly eventBusInstance: MessageEventBus;
+
 	__type: MessageEventBusDestinationTypeNames;
 
 	label: string;
@@ -24,7 +27,8 @@ export abstract class MessageEventBusDestination implements MessageEventBusDesti
 
 	anonymizeAuditMessages: boolean;
 
-	constructor(options: MessageEventBusDestinationOptions) {
+	constructor(eventBusInstance: MessageEventBus, options: MessageEventBusDestinationOptions) {
+		this.eventBusInstance = eventBusInstance;
 		this.id = !options.id || options.id.length !== 36 ? uuid() : options.id;
 		this.__type = options.__type ?? MessageEventBusDestinationTypeNames.abstract;
 		this.label = options.label ?? 'Log Destination';
@@ -37,15 +41,21 @@ export abstract class MessageEventBusDestination implements MessageEventBusDesti
 
 	startListening() {
 		if (this.enabled) {
-			eventBus.on(this.getId(), async (msg: EventMessageTypes) => {
-				await this.receiveFromEventBus(msg);
-			});
+			this.eventBusInstance.on(
+				this.getId(),
+				async (
+					msg: EventMessageTypes,
+					confirmCallback: (message: EventMessageTypes, src: EventMessageConfirmSource) => void,
+				) => {
+					await this.receiveFromEventBus({ msg, confirmCallback });
+				},
+			);
 			LoggerProxy.debug(`${this.id} listener started`);
 		}
 	}
 
 	stopListening() {
-		eventBus.removeAllListeners(this.getId());
+		this.eventBusInstance.removeAllListeners(this.getId());
 	}
 
 	enable() {
@@ -81,7 +91,6 @@ export abstract class MessageEventBusDestination implements MessageEventBusDesti
 			skipUpdateIfNoValuesChanged: true,
 			conflictPaths: ['id'],
 		});
-		Db.collections.EventDestinations.createQueryBuilder().insert().into('something').onConflict('');
 		return dbResult;
 	}
 
@@ -105,7 +114,7 @@ export abstract class MessageEventBusDestination implements MessageEventBusDesti
 		};
 	}
 
-	abstract receiveFromEventBus(msg: AbstractEventMessage): Promise<boolean>;
+	abstract receiveFromEventBus(emitterPayload: MessageWithCallback): Promise<boolean>;
 
 	toString() {
 		return JSON.stringify(this.serialize());

packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSentry.ee.ts

@@ -3,16 +3,15 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 import { MessageEventBusDestination } from './MessageEventBusDestination.ee';
 import * as Sentry from '@sentry/node';
-import { eventBus } from '../MessageEventBus/MessageEventBus';
+import { LoggerProxy, MessageEventBusDestinationTypeNames } from 'n8n-workflow';
 import type {
 	MessageEventBusDestinationOptions,
 	MessageEventBusDestinationSentryOptions,
 } from 'n8n-workflow';
-import { MessageEventBusDestinationTypeNames } from 'n8n-workflow';
 import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper';
-import type { EventMessageTypes } from '../EventMessageClasses';
 import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric';
 import { N8N_VERSION } from '@/constants';
+import type { MessageEventBus, MessageWithCallback } from '../MessageEventBus/MessageEventBus';
 
 export const isMessageEventBusDestinationSentryOptions = (
 	candidate: unknown,
@@ -34,8 +33,8 @@ export class MessageEventBusDestinationSentry
 
 	sentryClient?: Sentry.NodeClient;
 
-	constructor(options: MessageEventBusDestinationSentryOptions) {
-		super(options);
+	constructor(eventBusInstance: MessageEventBus, options: MessageEventBusDestinationSentryOptions) {
+		super(eventBusInstance, options);
 		this.label = options.label ?? 'Sentry DSN';
 		this.__type = options.__type ?? MessageEventBusDestinationTypeNames.sentry;
 		this.dsn = options.dsn;
@@ -54,7 +53,8 @@ export class MessageEventBusDestinationSentry
 		});
 	}
 
-	async receiveFromEventBus(msg: EventMessageTypes): Promise<boolean> {
+	async receiveFromEventBus(emitterPayload: MessageWithCallback): Promise<boolean> {
+		const { msg, confirmCallback } = emitterPayload;
 		let sendResult = false;
 		if (!this.sentryClient) return sendResult;
 		if (msg.eventName !== eventMessageGenericDestinationTestEvent) {
@@ -84,11 +84,12 @@ export class MessageEventBusDestinationSentry
 			);
 
 			if (sentryResult) {
-				eventBus.confirmSent(msg, { id: this.id, name: this.label });
+				// eventBus.confirmSent(msg, { id: this.id, name: this.label });
+				confirmCallback(msg, { id: this.id, name: this.label });
 				sendResult = true;
 			}
 		} catch (error) {
-			console.log(error);
+			if (error.message) LoggerProxy.debug(error.message as string);
 		}
 		return sendResult;
 	}
@@ -104,6 +105,7 @@ export class MessageEventBusDestinationSentry
 	}
 
 	static deserialize(
+		eventBusInstance: MessageEventBus,
 		data: MessageEventBusDestinationOptions,
 	): MessageEventBusDestinationSentry | null {
 		if (
@@ -111,7 +113,7 @@ export class MessageEventBusDestinationSentry
 			data.__type === MessageEventBusDestinationTypeNames.sentry &&
 			isMessageEventBusDestinationSentryOptions(data)
 		) {
-			return new MessageEventBusDestinationSentry(data);
+			return new MessageEventBusDestinationSentry(eventBusInstance, data);
 		}
 		return null;
 	}

packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSyslog.ee.ts

@@ -2,7 +2,6 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 import syslog from 'syslog-client';
-import { eventBus } from '../MessageEventBus/MessageEventBus';
 import type {
 	MessageEventBusDestinationOptions,
 	MessageEventBusDestinationSyslogOptions,
@@ -10,8 +9,8 @@ import type {
 import { LoggerProxy, MessageEventBusDestinationTypeNames } from 'n8n-workflow';
 import { MessageEventBusDestination } from './MessageEventBusDestination.ee';
 import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper';
-import type { EventMessageTypes } from '../EventMessageClasses';
 import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric';
+import type { MessageEventBus, MessageWithCallback } from '../MessageEventBus/MessageEventBus';
 
 export const isMessageEventBusDestinationSyslogOptions = (
 	candidate: unknown,
@@ -41,8 +40,8 @@ export class MessageEventBusDestinationSyslog
 
 	eol: string;
 
-	constructor(options: MessageEventBusDestinationSyslogOptions) {
-		super(options);
+	constructor(eventBusInstance: MessageEventBus, options: MessageEventBusDestinationSyslogOptions) {
+		super(eventBusInstance, options);
 		this.__type = options.__type ?? MessageEventBusDestinationTypeNames.syslog;
 		this.label = options.label ?? 'Syslog Server';
 
@@ -70,7 +69,8 @@ export class MessageEventBusDestinationSyslog
 		});
 	}
 
-	async receiveFromEventBus(msg: EventMessageTypes): Promise<boolean> {
+	async receiveFromEventBus(emitterPayload: MessageWithCallback): Promise<boolean> {
+		const { msg, confirmCallback } = emitterPayload;
 		let sendResult = false;
 		if (msg.eventName !== eventMessageGenericDestinationTestEvent) {
 			if (!isLogStreamingEnabled()) return sendResult;
@@ -92,16 +92,17 @@ export class MessageEventBusDestinationSyslog
 					timestamp: msg.ts.toJSDate(),
 				},
 				async (error) => {
-					if (error) {
-						console.log(error);
+					if (error?.message) {
+						LoggerProxy.debug(error.message);
 					} else {
-						eventBus.confirmSent(msg, { id: this.id, name: this.label });
+						// eventBus.confirmSent(msg, { id: this.id, name: this.label });
+						confirmCallback(msg, { id: this.id, name: this.label });
 						sendResult = true;
 					}
 				},
 			);
 		} catch (error) {
-			console.log(error);
+			if (error.message) LoggerProxy.debug(error.message as string);
 		}
 		if (msg.eventName === eventMessageGenericDestinationTestEvent) {
 			await new Promise((resolve) => setTimeout(resolve, 500));
@@ -124,6 +125,7 @@ export class MessageEventBusDestinationSyslog
 	}
 
 	static deserialize(
+		eventBusInstance: MessageEventBus,
 		data: MessageEventBusDestinationOptions,
 	): MessageEventBusDestinationSyslog | null {
 		if (
@@ -131,7 +133,7 @@ export class MessageEventBusDestinationSyslog
 			data.__type === MessageEventBusDestinationTypeNames.syslog &&
 			isMessageEventBusDestinationSyslogOptions(data)
 		) {
-			return new MessageEventBusDestinationSyslog(data);
+			return new MessageEventBusDestinationSyslog(eventBusInstance, data);
 		}
 		return null;
 	}

packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee.ts

@@ -1,4 +1,3 @@
-/* eslint-disable import/no-cycle */
 /* eslint-disable @typescript-eslint/no-unsafe-call */
 /* eslint-disable @typescript-eslint/no-unsafe-return */
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
@@ -6,23 +5,22 @@
 /* eslint-disable @typescript-eslint/no-unused-vars */
 /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
 import { MessageEventBusDestination } from './MessageEventBusDestination.ee';
-import type { AxiosRequestConfig, Method } from 'axios';
 import axios from 'axios';
-import { eventBus } from '../MessageEventBus/MessageEventBus';
-import type { EventMessageTypes } from '../EventMessageClasses';
+import type { AxiosRequestConfig, Method } from 'axios';
+import { jsonParse, LoggerProxy, MessageEventBusDestinationTypeNames } from 'n8n-workflow';
 import type {
 	MessageEventBusDestinationOptions,
 	MessageEventBusDestinationWebhookOptions,
 	MessageEventBusDestinationWebhookParameterItem,
 	MessageEventBusDestinationWebhookParameterOptions,
 } from 'n8n-workflow';
-import { jsonParse, LoggerProxy, MessageEventBusDestinationTypeNames } from 'n8n-workflow';
 import { CredentialsHelper } from '@/CredentialsHelper';
 import { UserSettings } from 'n8n-core';
 import { Agent as HTTPSAgent } from 'https';
 import config from '@/config';
 import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper';
 import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric';
+import type { MessageEventBus, MessageWithCallback } from '../MessageEventBus/MessageEventBus';
 
 export const isMessageEventBusDestinationWebhookOptions = (
 	candidate: unknown,
@@ -74,8 +72,11 @@ export class MessageEventBusDestinationWebhook
 
 	axiosRequestOptions: AxiosRequestConfig;
 
-	constructor(options: MessageEventBusDestinationWebhookOptions) {
-		super(options);
+	constructor(
+		eventBusInstance: MessageEventBus,
+		options: MessageEventBusDestinationWebhookOptions,
+	) {
+		super(eventBusInstance, options);
 		this.url = options.url;
 		this.label = options.label ?? 'Webhook Endpoint';
 		this.__type = options.__type ?? MessageEventBusDestinationTypeNames.webhook;
@@ -246,6 +247,7 @@ export class MessageEventBusDestinationWebhook
 	}
 
 	static deserialize(
+		eventBusInstance: MessageEventBus,
 		data: MessageEventBusDestinationOptions,
 	): MessageEventBusDestinationWebhook | null {
 		if (
@@ -253,12 +255,13 @@ export class MessageEventBusDestinationWebhook
 			data.__type === MessageEventBusDestinationTypeNames.webhook &&
 			isMessageEventBusDestinationWebhookOptions(data)
 		) {
-			return new MessageEventBusDestinationWebhook(data);
+			return new MessageEventBusDestinationWebhook(eventBusInstance, data);
 		}
 		return null;
 	}
 
-	async receiveFromEventBus(msg: EventMessageTypes): Promise<boolean> {
+	async receiveFromEventBus(emitterPayload: MessageWithCallback): Promise<boolean> {
+		const { msg, confirmCallback } = emitterPayload;
 		let sendResult = false;
 		if (msg.eventName !== eventMessageGenericDestinationTestEvent) {
 			if (!isLogStreamingEnabled()) return sendResult;
@@ -345,13 +348,13 @@ export class MessageEventBusDestinationWebhook
 			if (requestResponse) {
 				if (this.responseCodeMustMatch) {
 					if (requestResponse.status === this.expectedStatusCode) {
-						eventBus.confirmSent(msg, { id: this.id, name: this.label });
+						confirmCallback(msg, { id: this.id, name: this.label });
 						sendResult = true;
 					} else {
 						sendResult = false;
 					}
 				} else {
-					eventBus.confirmSent(msg, { id: this.id, name: this.label });
+					confirmCallback(msg, { id: this.id, name: this.label });
 					sendResult = true;
 				}
 			}

packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriter.ts

@@ -36,7 +36,7 @@ export interface MessageEventBusLogWriterOptions {
 interface ReadMessagesFromLogFileResult {
 	loggedMessages: EventMessageTypes[];
 	sentMessages: EventMessageTypes[];
-	unfinishedExecutions: Set<string>;
+	unfinishedExecutions: Record<string, EventMessageTypes[]>;
 }
 
 /**
@@ -156,7 +156,7 @@ export class MessageEventBusLogWriter {
 		const results: ReadMessagesFromLogFileResult = {
 			loggedMessages: [],
 			sentMessages: [],
-			unfinishedExecutions: new Set<string>(),
+			unfinishedExecutions: {},
 		};
 		const logCount = logHistory
 			? Math.min(config.get('eventBus.logWriter.keepLogCount') as number, logHistory)
@@ -188,14 +188,28 @@ export class MessageEventBusLogWriter {
 						if (isEventMessageOptions(json) && json.__type !== undefined) {
 							const msg = getEventMessageObjectByType(json);
 							if (msg !== null) results.loggedMessages.push(msg);
-							if (msg?.eventName === 'n8n.workflow.started' && msg?.payload?.executionId) {
-								results.unfinishedExecutions.add(msg?.payload?.executionId as string);
-							} else if (
-								(msg?.eventName === 'n8n.workflow.success' ||
-									msg?.eventName === 'n8n.workflow.failed') &&
-								msg?.payload?.executionId
-							) {
-								results.unfinishedExecutions.delete(msg?.payload?.executionId as string);
+							if (msg?.eventName && msg.payload?.executionId) {
+								const executionId = msg.payload.executionId as string;
+								switch (msg.eventName) {
+									case 'n8n.workflow.started':
+										if (!Object.keys(results.unfinishedExecutions).includes(executionId)) {
+											results.unfinishedExecutions[executionId] = [];
+										}
+										results.unfinishedExecutions[executionId] = [msg];
+										break;
+									case 'n8n.workflow.success':
+									case 'n8n.workflow.failed':
+									case 'n8n.workflow.crashed':
+										delete results.unfinishedExecutions[executionId];
+										break;
+									case 'n8n.node.started':
+									case 'n8n.node.finished':
+										if (!Object.keys(results.unfinishedExecutions).includes(executionId)) {
+											results.unfinishedExecutions[executionId] = [];
+										}
+										results.unfinishedExecutions[executionId].push(msg);
+										break;
+								}
 							}
 						}
 						if (isEventMessageConfirm(json) && mode !== 'all') {
@@ -204,9 +218,10 @@ export class MessageEventBusLogWriter {
 								results.sentMessages.push(...removedMessage);
 							}
 						}
-					} catch {
+					} catch (error) {
 						LoggerProxy.error(
-							`Error reading line messages from file: ${logFileName}, line: ${line}`,
+							// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+							`Error reading line messages from file: ${logFileName}, line: ${line}, ${error.message}}`,
 						);
 					}
 				});
@@ -301,13 +316,13 @@ export class MessageEventBusLogWriter {
 		return (await this.getMessages('unsent')).loggedMessages;
 	}
 
-	async getUnfinishedExecutions(): Promise<Set<string>> {
+	async getUnfinishedExecutions(): Promise<Record<string, EventMessageTypes[]>> {
 		return (await this.getMessages('unfinished')).unfinishedExecutions;
 	}
 
 	async getUnsentAndUnfinishedExecutions(): Promise<{
 		unsentMessages: EventMessageTypes[];
-		unfinishedExecutions: Set<string>;
+		unfinishedExecutions: Record<string, EventMessageTypes[]>;
 	}> {
 		const result = await this.getMessages('unsent');
 		return {

packages/cli/src/eventbus/eventBusRoutes.ts

@@ -30,6 +30,7 @@ import type { User } from '../databases/entities/User';
 import * as ResponseHelper from '@/ResponseHelper';
 import type { EventMessageNodeOptions } from './EventMessageClasses/EventMessageNode';
 import { EventMessageNode } from './EventMessageClasses/EventMessageNode';
+import { recoverExecutionDataFromEventLogMessages } from './MessageEventBus/recoverEvents';
 
 export const eventBusRouter = express.Router();
 
@@ -102,6 +103,32 @@ eventBusRouter.get(
 	}),
 );
 
+eventBusRouter.get(
+	'/execution-recover/:id',
+	ResponseHelper.send(async (req: express.Request): Promise<any> => {
+		if (req.params?.id) {
+			let logHistory;
+			let applyToDb = true;
+			if (req.query?.logHistory) {
+				logHistory = parseInt(req.query.logHistory as string, 10);
+			}
+			if (req.query?.applyToDb) {
+				applyToDb = !!req.query.applyToDb;
+			}
+			const messages = await eventBus.getEventsByExecutionId(req.params.id, logHistory);
+			if (messages.length > 0) {
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+				const recoverResult = await recoverExecutionDataFromEventLogMessages(
+					req.params.id,
+					messages,
+					applyToDb,
+				);
+				return recoverResult;
+			}
+		}
+	}),
+);
+
 eventBusRouter.post(
 	'/event',
 	ResponseHelper.send(async (req: express.Request): Promise<any> => {
@@ -159,17 +186,23 @@ eventBusRouter.post(
 			switch (req.body.__type) {
 				case MessageEventBusDestinationTypeNames.sentry:
 					if (isMessageEventBusDestinationSentryOptions(req.body)) {
-						result = await eventBus.addDestination(new MessageEventBusDestinationSentry(req.body));
+						result = await eventBus.addDestination(
+							new MessageEventBusDestinationSentry(eventBus, req.body),
+						);
 					}
 					break;
 				case MessageEventBusDestinationTypeNames.webhook:
 					if (isMessageEventBusDestinationWebhookOptions(req.body)) {
-						result = await eventBus.addDestination(new MessageEventBusDestinationWebhook(req.body));
+						result = await eventBus.addDestination(
+							new MessageEventBusDestinationWebhook(eventBus, req.body),
+						);
 					}
 					break;
 				case MessageEventBusDestinationTypeNames.syslog:
 					if (isMessageEventBusDestinationSyslogOptions(req.body)) {
-						result = await eventBus.addDestination(new MessageEventBusDestinationSyslog(req.body));
+						result = await eventBus.addDestination(
+							new MessageEventBusDestinationSyslog(eventBus, req.body),
+						);
 					}
 					break;
 				default:
@@ -180,7 +213,10 @@ eventBusRouter.post(
 			}
 			if (result) {
 				await result.saveToDb();
-				return result;
+				return {
+					...result,
+					eventBusInstance: undefined,
+				};
 			}
 			throw new BadRequestError('There was an error adding the destination');
 		}

packages/cli/src/executions/executions.service.ts

@@ -3,7 +3,15 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 import { validate as jsonSchemaValidate } from 'jsonschema';
 import { BinaryDataManager } from 'n8n-core';
-import type { IDataObject, IWorkflowBase, JsonObject } from 'n8n-workflow';
+import type {
+	IDataObject,
+	IWorkflowBase,
+	JsonObject,
+	ExecutionStatus,
+	IRunExecutionData,
+	NodeOperationError,
+	IExecutionsSummary,
+} from 'n8n-workflow';
 import { deepCopy, LoggerProxy, jsonParse, Workflow } from 'n8n-workflow';
 import type { FindOperator, FindOptionsWhere } from 'typeorm';
 import { In, IsNull, LessThanOrEqual, Not, Raw } from 'typeorm';
@@ -25,6 +33,7 @@ import { getSharedWorkflowIds } from '@/WorkflowHelpers';
 import { WorkflowRunner } from '@/WorkflowRunner';
 import * as Db from '@/Db';
 import * as GenericHelpers from '@/GenericHelpers';
+import { parse } from 'flatted';
 
 interface IGetExecutionsQueryFilter {
 	id?: FindOperator<string>;
@@ -32,6 +41,7 @@ interface IGetExecutionsQueryFilter {
 	mode?: string;
 	retryOf?: string;
 	retrySuccessId?: string;
+	status?: ExecutionStatus[];
 	workflowId?: string;
 	// eslint-disable-next-line @typescript-eslint/no-explicit-any
 	waitTill?: FindOperator<any> | boolean;
@@ -45,6 +55,10 @@ const schemaGetExecutionsQueryFilter = {
 		mode: { type: 'string' },
 		retryOf: { type: 'string' },
 		retrySuccessId: { type: 'string' },
+		status: {
+			type: 'array',
+			items: { type: 'string' },
+		},
 		waitTill: { type: 'boolean' },
 		workflowId: { anyOf: [{ type: 'integer' }, { type: 'string' }] },
 	},
@@ -193,7 +207,16 @@ export class ExecutionsService {
 				.map(({ id }) => id),
 		);
 
-		const findWhere: FindOptionsWhere<ExecutionEntity> = { workflowId: In(sharedWorkflowIds) };
+		const findWhere: FindOptionsWhere<ExecutionEntity> = {
+			workflowId: In(sharedWorkflowIds),
+		};
+		if (filter?.status) {
+			Object.assign(findWhere, { status: In(filter.status) });
+			delete filter.status; // remove status from filter so it does not get applied twice
+		}
+		if (filter?.finished) {
+			Object.assign(findWhere, { finished: filter.finished });
+		}
 
 		const rangeQuery: string[] = [];
 		const rangeQueryParams: {
@@ -257,7 +280,42 @@ export class ExecutionsService {
 			req.user,
 		);
 
-		const formattedExecutions = executions.map((execution) => {
+		const formattedExecutions: IExecutionsSummary[] = executions.map((execution) => {
+			// inject potential node execution errors into the execution response
+			const nodeExecutionStatus = {};
+			let lastNodeExecuted;
+			let executionError;
+			try {
+				const data = parse(execution.data) as IRunExecutionData;
+				lastNodeExecuted = data?.resultData?.lastNodeExecuted ?? '';
+				executionError = data?.resultData?.error;
+				if (data?.resultData?.runData) {
+					for (const key of Object.keys(data.resultData.runData)) {
+						const errors = data.resultData.runData[key]
+							?.filter((taskdata) => taskdata.error?.name)
+							?.map((taskdata) => {
+								if (taskdata.error?.name === 'NodeOperationError') {
+									return {
+										name: (taskdata.error as NodeOperationError).name,
+										message: (taskdata.error as NodeOperationError).message,
+										description: (taskdata.error as NodeOperationError).description,
+									};
+								} else {
+									return {
+										name: taskdata.error?.name,
+									};
+								}
+							});
+						Object.assign(nodeExecutionStatus, {
+							[key]: {
+								executionStatus: data.resultData.runData[key][0].executionStatus,
+								errors,
+								data: data.resultData.runData[key][0].data ?? undefined,
+							},
+						});
+					}
+				}
+			} catch {}
 			return {
 				id: execution.id,
 				finished: execution.finished,
@@ -269,9 +327,12 @@ export class ExecutionsService {
 				stoppedAt: execution.stoppedAt,
 				workflowId: execution.workflowData?.id ?? '',
 				workflowName: execution.workflowData?.name,
-			};
+				status: execution.status,
+				lastNodeExecuted,
+				executionError,
+				nodeExecutionStatus,
+			} as IExecutionsSummary;
 		});
-
 		return {
 			count,
 			results: formattedExecutions,

packages/cli/src/push/abstract.push.ts

@@ -1,5 +1,6 @@
 import { LoggerProxy as Logger } from 'n8n-workflow';
 import type { IPushDataType } from '@/Interfaces';
+import { eventBus } from '../eventbus';
 
 export abstract class AbstractPush<T> {
 	protected connections: Record<string, T> = {};
@@ -10,6 +11,7 @@ export abstract class AbstractPush<T> {
 	protected add(sessionId: string, connection: T): void {
 		const { connections } = this;
 		Logger.debug('Add editor-UI session', { sessionId });
+		eventBus.emit('editorUiConnected', sessionId);
 
 		const existingConnection = connections[sessionId];
 		if (existingConnection) {

packages/cli/test/integration/eventbus.test.ts

@@ -22,6 +22,7 @@ import { MessageEventBusDestinationWebhook } from '@/eventbus/MessageEventBusDes
 import { MessageEventBusDestinationSentry } from '@/eventbus/MessageEventBusDestination/MessageEventBusDestinationSentry.ee';
 import { EventMessageAudit } from '@/eventbus/EventMessageClasses/EventMessageAudit';
 import { v4 as uuid } from 'uuid';
+import { EventNamesTypes } from '../../src/eventbus/EventMessageClasses';
 
 jest.unmock('@/eventbus/MessageEventBus/MessageEventBus');
 jest.mock('axios');
@@ -62,12 +63,6 @@ const testSentryDestination: MessageEventBusDestinationSentryOptions = {
 	subscribedEvents: ['n8n.test.message', 'n8n.audit.user.updated'],
 };
 
-async function cleanLogs() {
-	eventBus.logWriter.cleanAllLogs();
-	const allMessages = await eventBus.getEventsAll();
-	expect(allMessages.length).toBe(0);
-}
-
 async function confirmIdInAll(id: string) {
 	const sent = await eventBus.getEventsAll();
 	expect(sent.length).toBeGreaterThan(0);
@@ -106,13 +101,13 @@ beforeAll(async () => {
 	config.set('eventBus.logWriter.logBaseName', 'n8n-test-logwriter');
 	config.set('eventBus.logWriter.keepLogCount', '1');
 	config.set('enterprise.features.logStreaming', true);
+
 	await eventBus.initialize();
 });
 
 beforeEach(async () => {
 	config.set('userManagement.disabled', false);
 	config.set('userManagement.isInstanceOwnerSetUp', true);
-	config.set('enterprise.features.logStreaming', false);
 });
 
 afterAll(async () => {
@@ -127,7 +122,10 @@ test('should have a running logwriter process', () => {
 });
 
 test('should have logwriter log messages', async () => {
-	const testMessage = new EventMessageGeneric({ eventName: 'n8n.test.message', id: uuid() });
+	const testMessage = new EventMessageGeneric({
+		eventName: 'n8n.test.message' as EventNamesTypes,
+		id: uuid(),
+	});
 	await eventBus.send(testMessage);
 	await new Promise((resolve) => {
 		eventBus.logWriter.worker?.once('message', async (msg: { command: string; data: any }) => {
@@ -176,10 +174,13 @@ test('GET /eventbus/destination all returned destinations should exist in eventb
 	}
 });
 
+// this test (presumably the mocking) is causing the test suite to randomly fail
 test.skip('should send message to syslog', async () => {
-	const testMessage = new EventMessageGeneric({ eventName: 'n8n.test.message', id: uuid() });
+	const testMessage = new EventMessageGeneric({
+		eventName: 'n8n.test.message' as EventNamesTypes,
+		id: uuid(),
+	});
 	config.set('enterprise.features.logStreaming', true);
-	// await cleanLogs();
 
 	const syslogDestination = eventBus.destinations[
 		testSyslogDestination.id!
@@ -217,11 +218,10 @@ test.skip('should send message to syslog', async () => {
 
 test.skip('should confirm send message if there are no subscribers', async () => {
 	const testMessageUnsubscribed = new EventMessageGeneric({
-		eventName: 'n8n.test.unsub',
+		eventName: 'n8n.test.unsub' as EventNamesTypes,
 		id: uuid(),
 	});
 	config.set('enterprise.features.logStreaming', true);
-	// await cleanLogs();
 
 	const syslogDestination = eventBus.destinations[
 		testSyslogDestination.id!
@@ -229,11 +229,6 @@ test.skip('should confirm send message if there are no subscribers', async () =>
 
 	syslogDestination.enable();
 
-	const mockedSyslogClientLog = jest.spyOn(syslogDestination.client, 'log');
-	mockedSyslogClientLog.mockImplementation((_m, _options, _cb) => {
-		return syslogDestination.client;
-	});
-
 	await eventBus.send(testMessageUnsubscribed);
 
 	await new Promise((resolve) => {
@@ -244,7 +239,6 @@ test.skip('should confirm send message if there are no subscribers', async () =>
 					await confirmIdInAll(testMessageUnsubscribed.id);
 				} else if (msg.command === 'confirmMessageSent') {
 					await confirmIdSent(testMessageUnsubscribed.id);
-					expect(mockedSyslogClientLog).toHaveBeenCalled();
 					syslogDestination.disable();
 					eventBus.logWriter.worker?.removeListener('message', handler002);
 					resolve(true);
@@ -264,7 +258,6 @@ test('should anonymize audit message to syslog ', async () => {
 		id: uuid(),
 	});
 	config.set('enterprise.features.logStreaming', true);
-	// await cleanLogs();
 
 	const syslogDestination = eventBus.destinations[
 		testSyslogDestination.id!
@@ -322,9 +315,11 @@ test('should anonymize audit message to syslog ', async () => {
 });
 
 test('should send message to webhook ', async () => {
-	const testMessage = new EventMessageGeneric({ eventName: 'n8n.test.message', id: uuid() });
+	const testMessage = new EventMessageGeneric({
+		eventName: 'n8n.test.message' as EventNamesTypes,
+		id: uuid(),
+	});
 	config.set('enterprise.features.logStreaming', true);
-	// await cleanLogs();
 
 	const webhookDestination = eventBus.destinations[
 		testWebhookDestination.id!
@@ -355,9 +350,11 @@ test('should send message to webhook ', async () => {
 });
 
 test('should send message to sentry ', async () => {
-	const testMessage = new EventMessageGeneric({ eventName: 'n8n.test.message', id: uuid() });
+	const testMessage = new EventMessageGeneric({
+		eventName: 'n8n.test.message' as EventNamesTypes,
+		id: uuid(),
+	});
 	config.set('enterprise.features.logStreaming', true);
-	// await cleanLogs();
 
 	const sentryDestination = eventBus.destinations[
 		testSentryDestination.id!

packages/cli/test/unit/ActiveExecutions.test.ts

@@ -156,6 +156,7 @@ function mockFullRunData(): IRun {
 		},
 		mode: 'manual',
 		startedAt: new Date(),
+		status: 'new',
 	};
 }