feat: Add global event bus (#4860)

* fix branch

* fix deserialize, add filewriter

* add catchAll eventGroup/Name

* adding simple Redis sender and receiver to eventbus

* remove native node threads

* improve eventbus

* refactor and simplify

* more refactoring and syslog client

* more refactor, improved endpoints and eventbus

* remove local broker and receivers from mvp

* destination de/serialization

* create MessageEventBusDestinationEntity

* db migrations, load destinations at startup

* add delete destination endpoint

* pnpm merge and circular import fix

* delete destination fix

* trigger log file shuffle after size reached

* add environment variables for eventbus

* reworking event messages

* serialize to thread fix

* some refactor and lint fixing

* add emit to eventbus

* cleanup and fix sending unsent

* quicksave frontend trial

* initial EventTree vue component

* basic log streaming settings in vue

* http request code merge

* create destination settings modals

* fix eventmessage options types

* credentials are loaded

* fix and clean up frontend code

* move request code to axios

* update lock file

* merge fix

* fix redis build

* move destination interfaces into workflow pkg

* revive sentry as destination

* migration fixes and frontend cleanup

* N8N-5777 / N8N-5789 N8N-5788

* N8N-5784

* N8N-5782 removed event levels

* N8N-5790 sentry destination cleanup

* N8N-5786 and refactoring

* N8N-5809 and refactor/cleanup

* UI fixes and anonymize renaming

* N8N-5837

* N8N-5834

* fix no-items UI issues

* remove card / settings label in modal

* N8N-5842 fix

* disable webhook auth for now and update ui

* change sidebar to tabs

* remove payload option

* extend audit events with more user data

* N8N-5853 and UI revert to sidebar

* remove redis destination

* N8N-5864 / N8N-5868 / N8N-5867 / N8N-5865

* ui and licensing fixes

* add node events and info bubbles to frontend

* ui wording changes

* frontend tests

* N8N-5896 and ee rename

* improves backend tests

* merge fix

* fix backend test

* make linter happy

* remove unnecessary cfg / limit  actions to owners

* fix multiple sentry DSN and anon bug

* eslint fix

* more tests and fixes

* merge fix

* fix workflow audit events

* remove 'n8n.workflow.execution.error' event

* merge fix

* lint fix

* lint fix

* review fixes

* fix merge

* prettier fixes

* merge

* review changes

* use loggerproxy

* remove catch from internal hook promises

* fix tests

* lint fix

* include review PR changes

* review changes

* delete duplicate lines from a bad merge

* decouple log-streaming UI options from public API

* logstreaming -> log-streaming for consistency

* do not make unnecessary api calls when log streaming is disabled

* prevent sentryClient.close() from being called if init failed

* fix the e2e test for log-streaming

* review changes

* cleanup

* use `private` for one last private property

* do not use node prefix package names.. just yet

* remove unused import

* fix the tests

because there is a folder called `events`, tsc-alias is messing up all imports for native events module.
https://github.com/justkey007/tsc-alias/issues/152

Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ <aditya@netroy.in>

packages/cli/src/eventbus/EventMessageClasses/AbstractEventMessage.ts

@@ -0,0 +1,143 @@
+/* eslint-disable @typescript-eslint/no-unsafe-argument */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { DateTime } from 'luxon';
+import type { EventMessageTypeNames, JsonObject } from 'n8n-workflow';
+import { v4 as uuid } from 'uuid';
+import type { AbstractEventPayload } from './AbstractEventPayload';
+import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
+
+function modifyUnderscoredKeys(
+	input: { [key: string]: any },
+	modifier: (secret: string) => string | undefined = () => '*',
+) {
+	const result: { [key: string]: any } = {};
+	if (!input) return input;
+	Object.keys(input).forEach((key) => {
+		if (typeof input[key] === 'string') {
+			if (key.substring(0, 1) === '_') {
+				const modifierResult = modifier(input[key]);
+				if (modifierResult !== undefined) {
+					result[key] = modifier(input[key]);
+				}
+			} else {
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+				result[key] = input[key];
+			}
+		} else if (typeof input[key] === 'object') {
+			if (Array.isArray(input[key])) {
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
+				result[key] = input[key].map((item: any) => {
+					if (typeof item === 'object' && !Array.isArray(item)) {
+						return modifyUnderscoredKeys(item, modifier);
+					} else {
+						// eslint-disable-next-line @typescript-eslint/no-unsafe-return
+						return item;
+					}
+				});
+			} else {
+				result[key] = modifyUnderscoredKeys(input[key], modifier);
+			}
+		} else {
+			// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+			result[key] = input[key];
+		}
+	});
+
+	return result;
+}
+
+export const isEventMessage = (candidate: unknown): candidate is AbstractEventMessage => {
+	const o = candidate as AbstractEventMessage;
+	if (!o) return false;
+	return (
+		o.eventName !== undefined &&
+		o.id !== undefined &&
+		o.ts !== undefined &&
+		o.getEventName !== undefined
+	);
+};
+
+export const isEventMessageOptions = (
+	candidate: unknown,
+): candidate is AbstractEventMessageOptions => {
+	const o = candidate as AbstractEventMessageOptions;
+	if (!o) return false;
+	if (o.eventName !== undefined) {
+		if (o.eventName.match(/^[\w\s]+\.[\w\s]+\.[\w\s]+/)) {
+			return true;
+		}
+	}
+	return false;
+};
+
+export const isEventMessageOptionsWithType = (
+	candidate: unknown,
+	expectedType: string,
+): candidate is AbstractEventMessageOptions => {
+	const o = candidate as AbstractEventMessageOptions;
+	if (!o) return false;
+	return o.eventName !== undefined && o.__type !== undefined && o.__type === expectedType;
+};
+
+export abstract class AbstractEventMessage {
+	abstract readonly __type: EventMessageTypeNames;
+
+	id: string;
+
+	ts: DateTime;
+
+	eventName: string;
+
+	message: string;
+
+	abstract payload: AbstractEventPayload;
+
+	/**
+	 * Creates a new instance of Event Message
+	 * @param props.eventName The specific events name e.g. "n8n.workflow.workflowStarted"
+	 * @param props.level The log level, defaults to. "info"
+	 * @param props.severity The severity of the event e.g. "normal"
+	 * @returns instance of EventMessage
+	 */
+	constructor(options: AbstractEventMessageOptions) {
+		this.setOptionsOrDefault(options);
+	}
+
+	abstract deserialize(data: JsonObject): this;
+	abstract setPayload(payload: AbstractEventPayload): this;
+
+	anonymize(): AbstractEventPayload {
+		const anonymizedPayload = modifyUnderscoredKeys(this.payload);
+		return anonymizedPayload;
+	}
+
+	serialize(): AbstractEventMessageOptions {
+		return {
+			__type: this.__type,
+			id: this.id,
+			ts: this.ts.toISO(),
+			eventName: this.eventName,
+			message: this.message,
+			payload: this.payload,
+		};
+	}
+
+	setOptionsOrDefault(options: AbstractEventMessageOptions) {
+		this.id = options.id ?? uuid();
+		this.eventName = options.eventName;
+		this.message = options.message ?? options.eventName;
+		if (typeof options.ts === 'string') {
+			this.ts = DateTime.fromISO(options.ts) ?? DateTime.now();
+		} else {
+			this.ts = options.ts ?? DateTime.now();
+		}
+	}
+
+	getEventName(): string {
+		return this.eventName;
+	}
+
+	toString() {
+		return JSON.stringify(this.serialize());
+	}
+}

packages/cli/src/eventbus/EventMessageClasses/AbstractEventMessageOptions.ts

@@ -0,0 +1,13 @@
+import type { DateTime } from 'luxon';
+import { EventMessageTypeNames } from 'n8n-workflow';
+import type { AbstractEventPayload } from './AbstractEventPayload';
+
+export interface AbstractEventMessageOptions {
+	__type?: EventMessageTypeNames;
+	id?: string;
+	ts?: DateTime | string;
+	eventName: string;
+	message?: string;
+	payload?: AbstractEventPayload;
+	anonymize?: boolean;
+}

packages/cli/src/eventbus/EventMessageClasses/AbstractEventPayload.ts

@@ -0,0 +1,5 @@
+import type { IWorkflowBase, JsonValue } from 'n8n-workflow';
+
+export interface AbstractEventPayload {
+	[key: string]: JsonValue | IWorkflowBase | undefined;
+}

packages/cli/src/eventbus/EventMessageClasses/EventMessageAudit.ts

@@ -0,0 +1,74 @@
+import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage';
+import { EventMessageTypeNames, JsonObject, JsonValue } from 'n8n-workflow';
+import { AbstractEventPayload } from './AbstractEventPayload';
+import { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
+
+export const eventNamesAudit = [
+	'n8n.audit.user.signedup',
+	'n8n.audit.user.updated',
+	'n8n.audit.user.deleted',
+	'n8n.audit.user.invited',
+	'n8n.audit.user.invitation.accepted',
+	'n8n.audit.user.reinvited',
+	'n8n.audit.user.email.failed',
+	'n8n.audit.user.reset.requested',
+	'n8n.audit.user.reset',
+	'n8n.audit.user.credentials.created',
+	'n8n.audit.user.credentials.shared',
+	'n8n.audit.user.api.created',
+	'n8n.audit.user.api.deleted',
+	'n8n.audit.package.installed',
+	'n8n.audit.package.updated',
+	'n8n.audit.package.deleted',
+	'n8n.audit.workflow.created',
+	'n8n.audit.workflow.deleted',
+	'n8n.audit.workflow.updated',
+] as const;
+export type EventNamesAuditType = typeof eventNamesAudit[number];
+
+// --------------------------------------
+// EventMessage class for Audit events
+// --------------------------------------
+export interface EventPayloadAudit extends AbstractEventPayload {
+	msg?: JsonValue;
+	userId?: string;
+	userEmail?: string;
+	firstName?: string;
+	lastName?: string;
+}
+
+export interface EventMessageAuditOptions extends AbstractEventMessageOptions {
+	eventName: EventNamesAuditType;
+
+	payload?: EventPayloadAudit;
+}
+
+export class EventMessageAudit extends AbstractEventMessage {
+	// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
+	readonly __type = EventMessageTypeNames.audit;
+
+	eventName: EventNamesAuditType;
+
+	payload: EventPayloadAudit;
+
+	constructor(options: EventMessageAuditOptions) {
+		super(options);
+		if (options.payload) this.setPayload(options.payload);
+		if (options.anonymize) {
+			this.anonymize();
+		}
+	}
+
+	setPayload(payload: EventPayloadAudit): this {
+		this.payload = payload;
+		return this;
+	}
+
+	deserialize(data: JsonObject): this {
+		if (isEventMessageOptionsWithType(data, this.__type)) {
+			this.setOptionsOrDefault(data);
+			if (data.payload) this.setPayload(data.payload as EventPayloadAudit);
+		}
+		return this;
+	}
+}

packages/cli/src/eventbus/EventMessageClasses/EventMessageConfirm.ts

@@ -0,0 +1,39 @@
+import { DateTime } from 'luxon';
+import { EventMessageTypeNames, JsonObject, JsonValue } from 'n8n-workflow';
+
+export interface EventMessageConfirmSource extends JsonObject {
+	id: string;
+	name: string;
+}
+
+export class EventMessageConfirm {
+	readonly __type = EventMessageTypeNames.confirm;
+
+	readonly confirm: string;
+
+	readonly source?: EventMessageConfirmSource;
+
+	readonly ts: DateTime;
+
+	constructor(confirm: string, source?: EventMessageConfirmSource) {
+		this.confirm = confirm;
+		this.ts = DateTime.now();
+		if (source) this.source = source;
+	}
+
+	serialize(): JsonValue {
+		// TODO: filter payload for sensitive info here?
+		return {
+			__type: this.__type,
+			confirm: this.confirm,
+			ts: this.ts.toISO(),
+			source: this.source ?? { name: '', id: '' },
+		};
+	}
+}
+
+export const isEventMessageConfirm = (candidate: unknown): candidate is EventMessageConfirm => {
+	const o = candidate as EventMessageConfirm;
+	if (!o) return false;
+	return o.confirm !== undefined && o.ts !== undefined;
+};

packages/cli/src/eventbus/EventMessageClasses/EventMessageGeneric.ts

@@ -0,0 +1,41 @@
+import { EventMessageTypeNames, JsonObject } from 'n8n-workflow';
+import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage';
+import type { AbstractEventPayload } from './AbstractEventPayload';
+import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
+
+export const eventMessageGenericDestinationTestEvent = 'n8n.destination.test';
+
+export interface EventPayloadGeneric extends AbstractEventPayload {
+	msg?: string;
+}
+
+export interface EventMessageGenericOptions extends AbstractEventMessageOptions {
+	payload?: EventPayloadGeneric;
+}
+
+export class EventMessageGeneric extends AbstractEventMessage {
+	readonly __type = EventMessageTypeNames.generic;
+
+	payload: EventPayloadGeneric;
+
+	constructor(options: EventMessageGenericOptions) {
+		super(options);
+		if (options.payload) this.setPayload(options.payload);
+		if (options.anonymize) {
+			this.anonymize();
+		}
+	}
+
+	setPayload(payload: EventPayloadGeneric): this {
+		this.payload = payload;
+		return this;
+	}
+
+	deserialize(data: JsonObject): this {
+		if (isEventMessageOptionsWithType(data, this.__type)) {
+			this.setOptionsOrDefault(data);
+			if (data.payload) this.setPayload(data.payload as EventPayloadGeneric);
+		}
+		return this;
+	}
+}

packages/cli/src/eventbus/EventMessageClasses/EventMessageNode.ts

@@ -0,0 +1,49 @@
+import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage';
+import { EventMessageTypeNames, JsonObject } from 'n8n-workflow';
+import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
+import type { AbstractEventPayload } from './AbstractEventPayload';
+
+export const eventNamesNode = ['n8n.node.started', 'n8n.node.finished'] as const;
+export type EventNamesNodeType = typeof eventNamesNode[number];
+
+// --------------------------------------
+// EventMessage class for Node events
+// --------------------------------------
+export interface EventPayloadNode extends AbstractEventPayload {
+	msg?: string;
+}
+
+export interface EventMessageNodeOptions extends AbstractEventMessageOptions {
+	eventName: EventNamesNodeType;
+
+	payload?: EventPayloadNode | undefined;
+}
+
+export class EventMessageNode extends AbstractEventMessage {
+	readonly __type = EventMessageTypeNames.node;
+
+	eventName: EventNamesNodeType;
+
+	payload: EventPayloadNode;
+
+	constructor(options: EventMessageNodeOptions) {
+		super(options);
+		if (options.payload) this.setPayload(options.payload);
+		if (options.anonymize) {
+			this.anonymize();
+		}
+	}
+
+	setPayload(payload: EventPayloadNode): this {
+		this.payload = payload;
+		return this;
+	}
+
+	deserialize(data: JsonObject): this {
+		if (isEventMessageOptionsWithType(data, this.__type)) {
+			this.setOptionsOrDefault(data);
+			if (data.payload) this.setPayload(data.payload as EventPayloadNode);
+		}
+		return this;
+	}
+}

packages/cli/src/eventbus/EventMessageClasses/EventMessageWorkflow.ts

@@ -0,0 +1,61 @@
+import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage';
+import { EventMessageTypeNames, IWorkflowBase, JsonObject } from 'n8n-workflow';
+import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
+import type { AbstractEventPayload } from './AbstractEventPayload';
+import { IExecutionBase } from '@/Interfaces';
+
+export const eventNamesWorkflow = [
+	'n8n.workflow.started',
+	'n8n.workflow.success',
+	'n8n.workflow.failed',
+] as const;
+
+export type EventNamesWorkflowType = typeof eventNamesWorkflow[number];
+
+// --------------------------------------
+// EventMessage class for Workflow events
+// --------------------------------------
+interface EventPayloadWorkflow extends AbstractEventPayload {
+	msg?: string;
+
+	workflowData?: IWorkflowBase;
+
+	executionId?: IExecutionBase['id'];
+
+	workflowId?: IWorkflowBase['id'];
+}
+
+export interface EventMessageWorkflowOptions extends AbstractEventMessageOptions {
+	eventName: EventNamesWorkflowType;
+
+	payload?: EventPayloadWorkflow | undefined;
+}
+
+export class EventMessageWorkflow extends AbstractEventMessage {
+	readonly __type = EventMessageTypeNames.workflow;
+
+	eventName: EventNamesWorkflowType;
+
+	payload: EventPayloadWorkflow;
+
+	constructor(options: EventMessageWorkflowOptions) {
+		super(options);
+		if (options.payload) this.setPayload(options.payload);
+		if (options.anonymize) {
+			this.anonymize();
+		}
+	}
+
+	setPayload(payload: EventPayloadWorkflow): this {
+		this.payload = payload;
+		return this;
+	}
+
+	deserialize(data: JsonObject): this {
+		if (isEventMessageOptionsWithType(data, this.__type)) {
+			this.setOptionsOrDefault(data);
+			if (data.payload) this.setPayload(data.payload as EventPayloadWorkflow);
+		}
+		return this;
+	}
+}

packages/cli/src/eventbus/EventMessageClasses/Helpers.ts

@@ -0,0 +1,92 @@
+import type { EventMessageTypes } from '.';
+import { EventMessageGeneric, EventMessageGenericOptions } from './EventMessageGeneric';
+import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
+import { EventMessageWorkflow, EventMessageWorkflowOptions } from './EventMessageWorkflow';
+import { EventMessageTypeNames } from 'n8n-workflow';
+
+export const getEventMessageObjectByType = (
+	message: AbstractEventMessageOptions,
+): EventMessageTypes | null => {
+	switch (message.__type as EventMessageTypeNames) {
+		case EventMessageTypeNames.generic:
+			return new EventMessageGeneric(message as EventMessageGenericOptions);
+		case EventMessageTypeNames.workflow:
+			return new EventMessageWorkflow(message as EventMessageWorkflowOptions);
+		default:
+			return null;
+	}
+};
+
+interface StringIndexedObject {
+	[key: string]: StringIndexedObject | string;
+}
+
+export function eventGroupFromEventName(eventName: string): string | undefined {
+	const matches = eventName.match(/^[\w\s]+\.[\w\s]+/);
+	if (matches && matches?.length > 0) {
+		return matches[0];
+	}
+	return;
+}
+
+function dotsToObject2(dottedString: string, o?: StringIndexedObject): StringIndexedObject {
+	const rootObject: StringIndexedObject = o ?? {};
+	if (!dottedString) return rootObject;
+
+	const parts = dottedString.split('.'); /*?*/
+
+	let part: string | undefined;
+	let obj: StringIndexedObject = rootObject;
+	while ((part = parts.shift())) {
+		if (typeof obj[part] !== 'object') {
+			obj[part] = {
+				__name: part,
+			};
+		}
+		obj = obj[part] as StringIndexedObject;
+	}
+	return rootObject;
+}
+
+export function eventListToObject(dottedList: string[]): object {
+	const result = {};
+	dottedList.forEach((e) => {
+		dotsToObject2(e, result);
+	});
+	return result;
+}
+
+interface StringIndexedChild {
+	name: string;
+	children: StringIndexedChild[];
+}
+
+export function eventListToObjectTree(dottedList: string[]): StringIndexedChild {
+	const x: StringIndexedChild = {
+		name: 'eventTree',
+		children: [] as unknown as StringIndexedChild[],
+	};
+	dottedList.forEach((dottedString: string) => {
+		const parts = dottedString.split('.');
+
+		let part: string | undefined;
+		let children = x.children;
+		while ((part = parts.shift())) {
+			if (part) {
+				// eslint-disable-next-line @typescript-eslint/no-loop-func
+				const foundChild = children.find((e) => e.name === part);
+				if (foundChild) {
+					children = foundChild.children;
+				} else {
+					const newChild: StringIndexedChild = {
+						name: part,
+						children: [],
+					};
+					children.push(newChild);
+					children = newChild.children;
+				}
+			}
+		}
+	});
+	return x;
+}

packages/cli/src/eventbus/EventMessageClasses/index.ts

@@ -0,0 +1,17 @@
+import { EventMessageAudit, eventNamesAudit, EventNamesAuditType } from './EventMessageAudit';
+import { EventMessageGeneric } from './EventMessageGeneric';
+import { EventMessageNode, eventNamesNode, EventNamesNodeType } from './EventMessageNode';
+import {
+	EventMessageWorkflow,
+	eventNamesWorkflow,
+	EventNamesWorkflowType,
+} from './EventMessageWorkflow';
+
+export type EventNamesTypes = EventNamesAuditType | EventNamesWorkflowType | EventNamesNodeType;
+export const eventNamesAll = [...eventNamesAudit, ...eventNamesWorkflow, ...eventNamesNode];
+
+export type EventMessageTypes =
+	| EventMessageGeneric
+	| EventMessageWorkflow
+	| EventMessageAudit
+	| EventMessageNode;