feat(core): Add execution runData recovery and status field (#5112)

* adds ExecutionEvents view modal to ExecutionList

* fix time rendering and remove wf column

* checks for unfinished executions and fails them

* prevent re-setting stoppedAt for execution

* some cleanup / manually create rundata after crash

* quicksave

* remove Threads lib, log worker rewrite

* cleanup comment

* fix sentry destination return value

* test for tests...

* run tests with single worker

* fix tests

* remove console log

* add endpoint for execution data recovery

* lint cleanup and some refactoring

* fix accidental recursion

* remove cyclic imports

* add rundata recovery to Workflowrunner

* remove comments

* cleanup and refactor

* adds a status field to executions

* setExecutionStatus on queued worker

* fix onWorkflowPostExecute

* set waiting from worker

* get crashed status into frontend

* remove comment

* merge fix

* cleanup

* catch empty rundata in recovery

* refactor IExecutionsSummary and inject nodeExecution Errors

* reduce default event log size to 10mb from 100mb

* add per node execution status

* lint fix

* merge and lint fix

* phrasing change

* improve preview rendering and messaging

* remove debug

* Improve partial rundata recovery

* fix labels

* fix line through

* send manual rundata to ui at crash

* some type and msg push fixes

* improve recovered item rendering in preview

* update workflowStatistics on recover

* merge fix

* review fixes

* merge fix

* notify eventbus when ui is back up

* add a small timeout to make sure the UI is back up

* increase reconnect timeout to 30s

* adjust recover timeout and ui connection lost msg

* do not stop execution in editor after x reconnects

* add executionRecovered push event

* fix recovered connection not green

* remove reconnect toast and  merge existing rundata

* merge editor and recovered data for own mode

packages/cli/src/eventbus/MessageEventBusDestination/Helpers.ee.ts

@@ -1,5 +1,4 @@
-/* eslint-disable import/no-cycle */
-import type { EventDestinations } from '@db/entities/MessageEventBusDestinationEntity';
+import type { EventDestinations } from '@/databases/entities/MessageEventBusDestinationEntity';
 import { promClient } from '@/metrics';
 import {
 	EventMessageTypeNames,
@@ -12,21 +11,23 @@ import type { MessageEventBusDestination } from './MessageEventBusDestination.ee
 import { MessageEventBusDestinationSentry } from './MessageEventBusDestinationSentry.ee';
 import { MessageEventBusDestinationSyslog } from './MessageEventBusDestinationSyslog.ee';
 import { MessageEventBusDestinationWebhook } from './MessageEventBusDestinationWebhook.ee';
+import type { MessageEventBus } from '../MessageEventBus/MessageEventBus';
 
 export function messageEventBusDestinationFromDb(
+	eventBusInstance: MessageEventBus,
 	dbData: EventDestinations,
 ): MessageEventBusDestination | null {
 	const destinationData = dbData.destination;
 	if ('__type' in destinationData) {
 		switch (destinationData.__type) {
 			case MessageEventBusDestinationTypeNames.sentry:
-				return MessageEventBusDestinationSentry.deserialize(destinationData);
+				return MessageEventBusDestinationSentry.deserialize(eventBusInstance, destinationData);
 			case MessageEventBusDestinationTypeNames.syslog:
-				return MessageEventBusDestinationSyslog.deserialize(destinationData);
+				return MessageEventBusDestinationSyslog.deserialize(eventBusInstance, destinationData);
 			case MessageEventBusDestinationTypeNames.webhook:
-				return MessageEventBusDestinationWebhook.deserialize(destinationData);
+				return MessageEventBusDestinationWebhook.deserialize(eventBusInstance, destinationData);
 			default:
-				console.log('MessageEventBusDestination __type unknown');
+				LoggerProxy.debug('MessageEventBusDestination __type unknown');
 		}
 	}
 	return null;

packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestination.ee.ts

@@ -4,14 +4,17 @@ import { LoggerProxy, MessageEventBusDestinationTypeNames } from 'n8n-workflow';
 import * as Db from '@/Db';
 import type { AbstractEventMessage } from '../EventMessageClasses/AbstractEventMessage';
 import type { EventMessageTypes } from '../EventMessageClasses';
-import { eventBus } from '..';
 import type { DeleteResult, InsertResult } from 'typeorm';
+import type { EventMessageConfirmSource } from '../EventMessageClasses/EventMessageConfirm';
+import type { MessageEventBus, MessageWithCallback } from '../MessageEventBus/MessageEventBus';
 
 export abstract class MessageEventBusDestination implements MessageEventBusDestinationOptions {
 	// Since you can't have static abstract functions - this just serves as a reminder that you need to implement these. Please.
 	// static abstract deserialize(): MessageEventBusDestination | null;
 	readonly id: string;
 
+	readonly eventBusInstance: MessageEventBus;
+
 	__type: MessageEventBusDestinationTypeNames;
 
 	label: string;
@@ -24,7 +27,8 @@ export abstract class MessageEventBusDestination implements MessageEventBusDesti
 
 	anonymizeAuditMessages: boolean;
 
-	constructor(options: MessageEventBusDestinationOptions) {
+	constructor(eventBusInstance: MessageEventBus, options: MessageEventBusDestinationOptions) {
+		this.eventBusInstance = eventBusInstance;
 		this.id = !options.id || options.id.length !== 36 ? uuid() : options.id;
 		this.__type = options.__type ?? MessageEventBusDestinationTypeNames.abstract;
 		this.label = options.label ?? 'Log Destination';
@@ -37,15 +41,21 @@ export abstract class MessageEventBusDestination implements MessageEventBusDesti
 
 	startListening() {
 		if (this.enabled) {
-			eventBus.on(this.getId(), async (msg: EventMessageTypes) => {
-				await this.receiveFromEventBus(msg);
-			});
+			this.eventBusInstance.on(
+				this.getId(),
+				async (
+					msg: EventMessageTypes,
+					confirmCallback: (message: EventMessageTypes, src: EventMessageConfirmSource) => void,
+				) => {
+					await this.receiveFromEventBus({ msg, confirmCallback });
+				},
+			);
 			LoggerProxy.debug(`${this.id} listener started`);
 		}
 	}
 
 	stopListening() {
-		eventBus.removeAllListeners(this.getId());
+		this.eventBusInstance.removeAllListeners(this.getId());
 	}
 
 	enable() {
@@ -81,7 +91,6 @@ export abstract class MessageEventBusDestination implements MessageEventBusDesti
 			skipUpdateIfNoValuesChanged: true,
 			conflictPaths: ['id'],
 		});
-		Db.collections.EventDestinations.createQueryBuilder().insert().into('something').onConflict('');
 		return dbResult;
 	}
 
@@ -105,7 +114,7 @@ export abstract class MessageEventBusDestination implements MessageEventBusDesti
 		};
 	}
 
-	abstract receiveFromEventBus(msg: AbstractEventMessage): Promise<boolean>;
+	abstract receiveFromEventBus(emitterPayload: MessageWithCallback): Promise<boolean>;
 
 	toString() {
 		return JSON.stringify(this.serialize());

packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSentry.ee.ts

@@ -3,16 +3,15 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 import { MessageEventBusDestination } from './MessageEventBusDestination.ee';
 import * as Sentry from '@sentry/node';
-import { eventBus } from '../MessageEventBus/MessageEventBus';
+import { LoggerProxy, MessageEventBusDestinationTypeNames } from 'n8n-workflow';
 import type {
 	MessageEventBusDestinationOptions,
 	MessageEventBusDestinationSentryOptions,
 } from 'n8n-workflow';
-import { MessageEventBusDestinationTypeNames } from 'n8n-workflow';
 import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper';
-import type { EventMessageTypes } from '../EventMessageClasses';
 import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric';
 import { N8N_VERSION } from '@/constants';
+import type { MessageEventBus, MessageWithCallback } from '../MessageEventBus/MessageEventBus';
 
 export const isMessageEventBusDestinationSentryOptions = (
 	candidate: unknown,
@@ -34,8 +33,8 @@ export class MessageEventBusDestinationSentry
 
 	sentryClient?: Sentry.NodeClient;
 
-	constructor(options: MessageEventBusDestinationSentryOptions) {
-		super(options);
+	constructor(eventBusInstance: MessageEventBus, options: MessageEventBusDestinationSentryOptions) {
+		super(eventBusInstance, options);
 		this.label = options.label ?? 'Sentry DSN';
 		this.__type = options.__type ?? MessageEventBusDestinationTypeNames.sentry;
 		this.dsn = options.dsn;
@@ -54,7 +53,8 @@ export class MessageEventBusDestinationSentry
 		});
 	}
 
-	async receiveFromEventBus(msg: EventMessageTypes): Promise<boolean> {
+	async receiveFromEventBus(emitterPayload: MessageWithCallback): Promise<boolean> {
+		const { msg, confirmCallback } = emitterPayload;
 		let sendResult = false;
 		if (!this.sentryClient) return sendResult;
 		if (msg.eventName !== eventMessageGenericDestinationTestEvent) {
@@ -84,11 +84,12 @@ export class MessageEventBusDestinationSentry
 			);
 
 			if (sentryResult) {
-				eventBus.confirmSent(msg, { id: this.id, name: this.label });
+				// eventBus.confirmSent(msg, { id: this.id, name: this.label });
+				confirmCallback(msg, { id: this.id, name: this.label });
 				sendResult = true;
 			}
 		} catch (error) {
-			console.log(error);
+			if (error.message) LoggerProxy.debug(error.message as string);
 		}
 		return sendResult;
 	}
@@ -104,6 +105,7 @@ export class MessageEventBusDestinationSentry
 	}
 
 	static deserialize(
+		eventBusInstance: MessageEventBus,
 		data: MessageEventBusDestinationOptions,
 	): MessageEventBusDestinationSentry | null {
 		if (
@@ -111,7 +113,7 @@ export class MessageEventBusDestinationSentry
 			data.__type === MessageEventBusDestinationTypeNames.sentry &&
 			isMessageEventBusDestinationSentryOptions(data)
 		) {
-			return new MessageEventBusDestinationSentry(data);
+			return new MessageEventBusDestinationSentry(eventBusInstance, data);
 		}
 		return null;
 	}

packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSyslog.ee.ts

@@ -2,7 +2,6 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 import syslog from 'syslog-client';
-import { eventBus } from '../MessageEventBus/MessageEventBus';
 import type {
 	MessageEventBusDestinationOptions,
 	MessageEventBusDestinationSyslogOptions,
@@ -10,8 +9,8 @@ import type {
 import { LoggerProxy, MessageEventBusDestinationTypeNames } from 'n8n-workflow';
 import { MessageEventBusDestination } from './MessageEventBusDestination.ee';
 import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper';
-import type { EventMessageTypes } from '../EventMessageClasses';
 import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric';
+import type { MessageEventBus, MessageWithCallback } from '../MessageEventBus/MessageEventBus';
 
 export const isMessageEventBusDestinationSyslogOptions = (
 	candidate: unknown,
@@ -41,8 +40,8 @@ export class MessageEventBusDestinationSyslog
 
 	eol: string;
 
-	constructor(options: MessageEventBusDestinationSyslogOptions) {
-		super(options);
+	constructor(eventBusInstance: MessageEventBus, options: MessageEventBusDestinationSyslogOptions) {
+		super(eventBusInstance, options);
 		this.__type = options.__type ?? MessageEventBusDestinationTypeNames.syslog;
 		this.label = options.label ?? 'Syslog Server';
 
@@ -70,7 +69,8 @@ export class MessageEventBusDestinationSyslog
 		});
 	}
 
-	async receiveFromEventBus(msg: EventMessageTypes): Promise<boolean> {
+	async receiveFromEventBus(emitterPayload: MessageWithCallback): Promise<boolean> {
+		const { msg, confirmCallback } = emitterPayload;
 		let sendResult = false;
 		if (msg.eventName !== eventMessageGenericDestinationTestEvent) {
 			if (!isLogStreamingEnabled()) return sendResult;
@@ -92,16 +92,17 @@ export class MessageEventBusDestinationSyslog
 					timestamp: msg.ts.toJSDate(),
 				},
 				async (error) => {
-					if (error) {
-						console.log(error);
+					if (error?.message) {
+						LoggerProxy.debug(error.message);
 					} else {
-						eventBus.confirmSent(msg, { id: this.id, name: this.label });
+						// eventBus.confirmSent(msg, { id: this.id, name: this.label });
+						confirmCallback(msg, { id: this.id, name: this.label });
 						sendResult = true;
 					}
 				},
 			);
 		} catch (error) {
-			console.log(error);
+			if (error.message) LoggerProxy.debug(error.message as string);
 		}
 		if (msg.eventName === eventMessageGenericDestinationTestEvent) {
 			await new Promise((resolve) => setTimeout(resolve, 500));
@@ -124,6 +125,7 @@ export class MessageEventBusDestinationSyslog
 	}
 
 	static deserialize(
+		eventBusInstance: MessageEventBus,
 		data: MessageEventBusDestinationOptions,
 	): MessageEventBusDestinationSyslog | null {
 		if (
@@ -131,7 +133,7 @@ export class MessageEventBusDestinationSyslog
 			data.__type === MessageEventBusDestinationTypeNames.syslog &&
 			isMessageEventBusDestinationSyslogOptions(data)
 		) {
-			return new MessageEventBusDestinationSyslog(data);
+			return new MessageEventBusDestinationSyslog(eventBusInstance, data);
 		}
 		return null;
 	}

packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee.ts

@@ -1,4 +1,3 @@
-/* eslint-disable import/no-cycle */
 /* eslint-disable @typescript-eslint/no-unsafe-call */
 /* eslint-disable @typescript-eslint/no-unsafe-return */
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
@@ -6,23 +5,22 @@
 /* eslint-disable @typescript-eslint/no-unused-vars */
 /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
 import { MessageEventBusDestination } from './MessageEventBusDestination.ee';
-import type { AxiosRequestConfig, Method } from 'axios';
 import axios from 'axios';
-import { eventBus } from '../MessageEventBus/MessageEventBus';
-import type { EventMessageTypes } from '../EventMessageClasses';
+import type { AxiosRequestConfig, Method } from 'axios';
+import { jsonParse, LoggerProxy, MessageEventBusDestinationTypeNames } from 'n8n-workflow';
 import type {
 	MessageEventBusDestinationOptions,
 	MessageEventBusDestinationWebhookOptions,
 	MessageEventBusDestinationWebhookParameterItem,
 	MessageEventBusDestinationWebhookParameterOptions,
 } from 'n8n-workflow';
-import { jsonParse, LoggerProxy, MessageEventBusDestinationTypeNames } from 'n8n-workflow';
 import { CredentialsHelper } from '@/CredentialsHelper';
 import { UserSettings } from 'n8n-core';
 import { Agent as HTTPSAgent } from 'https';
 import config from '@/config';
 import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper';
 import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric';
+import type { MessageEventBus, MessageWithCallback } from '../MessageEventBus/MessageEventBus';
 
 export const isMessageEventBusDestinationWebhookOptions = (
 	candidate: unknown,
@@ -74,8 +72,11 @@ export class MessageEventBusDestinationWebhook
 
 	axiosRequestOptions: AxiosRequestConfig;
 
-	constructor(options: MessageEventBusDestinationWebhookOptions) {
-		super(options);
+	constructor(
+		eventBusInstance: MessageEventBus,
+		options: MessageEventBusDestinationWebhookOptions,
+	) {
+		super(eventBusInstance, options);
 		this.url = options.url;
 		this.label = options.label ?? 'Webhook Endpoint';
 		this.__type = options.__type ?? MessageEventBusDestinationTypeNames.webhook;
@@ -246,6 +247,7 @@ export class MessageEventBusDestinationWebhook
 	}
 
 	static deserialize(
+		eventBusInstance: MessageEventBus,
 		data: MessageEventBusDestinationOptions,
 	): MessageEventBusDestinationWebhook | null {
 		if (
@@ -253,12 +255,13 @@ export class MessageEventBusDestinationWebhook
 			data.__type === MessageEventBusDestinationTypeNames.webhook &&
 			isMessageEventBusDestinationWebhookOptions(data)
 		) {
-			return new MessageEventBusDestinationWebhook(data);
+			return new MessageEventBusDestinationWebhook(eventBusInstance, data);
 		}
 		return null;
 	}
 
-	async receiveFromEventBus(msg: EventMessageTypes): Promise<boolean> {
+	async receiveFromEventBus(emitterPayload: MessageWithCallback): Promise<boolean> {
+		const { msg, confirmCallback } = emitterPayload;
 		let sendResult = false;
 		if (msg.eventName !== eventMessageGenericDestinationTestEvent) {
 			if (!isLogStreamingEnabled()) return sendResult;
@@ -345,13 +348,13 @@ export class MessageEventBusDestinationWebhook
 			if (requestResponse) {
 				if (this.responseCodeMustMatch) {
 					if (requestResponse.status === this.expectedStatusCode) {
-						eventBus.confirmSent(msg, { id: this.id, name: this.label });
+						confirmCallback(msg, { id: this.id, name: this.label });
 						sendResult = true;
 					} else {
 						sendResult = false;
 					}
 				} else {
-					eventBus.confirmSent(msg, { id: this.id, name: this.label });
+					confirmCallback(msg, { id: this.id, name: this.label });
 					sendResult = true;
 				}
 			}