feat: Add global event bus (#4860)

* fix branch

* fix deserialize, add filewriter

* add catchAll eventGroup/Name

* adding simple Redis sender and receiver to eventbus

* remove native node threads

* improve eventbus

* refactor and simplify

* more refactoring and syslog client

* more refactor, improved endpoints and eventbus

* remove local broker and receivers from mvp

* destination de/serialization

* create MessageEventBusDestinationEntity

* db migrations, load destinations at startup

* add delete destination endpoint

* pnpm merge and circular import fix

* delete destination fix

* trigger log file shuffle after size reached

* add environment variables for eventbus

* reworking event messages

* serialize to thread fix

* some refactor and lint fixing

* add emit to eventbus

* cleanup and fix sending unsent

* quicksave frontend trial

* initial EventTree vue component

* basic log streaming settings in vue

* http request code merge

* create destination settings modals

* fix eventmessage options types

* credentials are loaded

* fix and clean up frontend code

* move request code to axios

* update lock file

* merge fix

* fix redis build

* move destination interfaces into workflow pkg

* revive sentry as destination

* migration fixes and frontend cleanup

* N8N-5777 / N8N-5789 N8N-5788

* N8N-5784

* N8N-5782 removed event levels

* N8N-5790 sentry destination cleanup

* N8N-5786 and refactoring

* N8N-5809 and refactor/cleanup

* UI fixes and anonymize renaming

* N8N-5837

* N8N-5834

* fix no-items UI issues

* remove card / settings label in modal

* N8N-5842 fix

* disable webhook auth for now and update ui

* change sidebar to tabs

* remove payload option

* extend audit events with more user data

* N8N-5853 and UI revert to sidebar

* remove redis destination

* N8N-5864 / N8N-5868 / N8N-5867 / N8N-5865

* ui and licensing fixes

* add node events and info bubbles to frontend

* ui wording changes

* frontend tests

* N8N-5896 and ee rename

* improves backend tests

* merge fix

* fix backend test

* make linter happy

* remove unnecessary cfg / limit  actions to owners

* fix multiple sentry DSN and anon bug

* eslint fix

* more tests and fixes

* merge fix

* fix workflow audit events

* remove 'n8n.workflow.execution.error' event

* merge fix

* lint fix

* lint fix

* review fixes

* fix merge

* prettier fixes

* merge

* review changes

* use loggerproxy

* remove catch from internal hook promises

* fix tests

* lint fix

* include review PR changes

* review changes

* delete duplicate lines from a bad merge

* decouple log-streaming UI options from public API

* logstreaming -> log-streaming for consistency

* do not make unnecessary api calls when log streaming is disabled

* prevent sentryClient.close() from being called if init failed

* fix the e2e test for log-streaming

* review changes

* cleanup

* use `private` for one last private property

* do not use node prefix package names.. just yet

* remove unused import

* fix the tests

because there is a folder called `events`, tsc-alias is messing up all imports for native events module.
https://github.com/justkey007/tsc-alias/issues/152

Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ <aditya@netroy.in>
This commit is contained in:
Michael Auerswald
2023-01-04 09:47:48 +01:00
committed by GitHub
parent 0795cdb74c
commit b67f803cbe
104 changed files with 5867 additions and 219 deletions

View File

@@ -0,0 +1,143 @@
/* eslint-disable @typescript-eslint/no-unsafe-argument */
/* eslint-disable @typescript-eslint/no-explicit-any */
import { DateTime } from 'luxon';
import type { EventMessageTypeNames, JsonObject } from 'n8n-workflow';
import { v4 as uuid } from 'uuid';
import type { AbstractEventPayload } from './AbstractEventPayload';
import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
function modifyUnderscoredKeys(
input: { [key: string]: any },
modifier: (secret: string) => string | undefined = () => '*',
) {
const result: { [key: string]: any } = {};
if (!input) return input;
Object.keys(input).forEach((key) => {
if (typeof input[key] === 'string') {
if (key.substring(0, 1) === '_') {
const modifierResult = modifier(input[key]);
if (modifierResult !== undefined) {
result[key] = modifier(input[key]);
}
} else {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
result[key] = input[key];
}
} else if (typeof input[key] === 'object') {
if (Array.isArray(input[key])) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
result[key] = input[key].map((item: any) => {
if (typeof item === 'object' && !Array.isArray(item)) {
return modifyUnderscoredKeys(item, modifier);
} else {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return item;
}
});
} else {
result[key] = modifyUnderscoredKeys(input[key], modifier);
}
} else {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
result[key] = input[key];
}
});
return result;
}
export const isEventMessage = (candidate: unknown): candidate is AbstractEventMessage => {
const o = candidate as AbstractEventMessage;
if (!o) return false;
return (
o.eventName !== undefined &&
o.id !== undefined &&
o.ts !== undefined &&
o.getEventName !== undefined
);
};
export const isEventMessageOptions = (
candidate: unknown,
): candidate is AbstractEventMessageOptions => {
const o = candidate as AbstractEventMessageOptions;
if (!o) return false;
if (o.eventName !== undefined) {
if (o.eventName.match(/^[\w\s]+\.[\w\s]+\.[\w\s]+/)) {
return true;
}
}
return false;
};
export const isEventMessageOptionsWithType = (
candidate: unknown,
expectedType: string,
): candidate is AbstractEventMessageOptions => {
const o = candidate as AbstractEventMessageOptions;
if (!o) return false;
return o.eventName !== undefined && o.__type !== undefined && o.__type === expectedType;
};
export abstract class AbstractEventMessage {
abstract readonly __type: EventMessageTypeNames;
id: string;
ts: DateTime;
eventName: string;
message: string;
abstract payload: AbstractEventPayload;
/**
* Creates a new instance of Event Message
* @param props.eventName The specific events name e.g. "n8n.workflow.workflowStarted"
* @param props.level The log level, defaults to. "info"
* @param props.severity The severity of the event e.g. "normal"
* @returns instance of EventMessage
*/
constructor(options: AbstractEventMessageOptions) {
this.setOptionsOrDefault(options);
}
abstract deserialize(data: JsonObject): this;
abstract setPayload(payload: AbstractEventPayload): this;
anonymize(): AbstractEventPayload {
const anonymizedPayload = modifyUnderscoredKeys(this.payload);
return anonymizedPayload;
}
serialize(): AbstractEventMessageOptions {
return {
__type: this.__type,
id: this.id,
ts: this.ts.toISO(),
eventName: this.eventName,
message: this.message,
payload: this.payload,
};
}
setOptionsOrDefault(options: AbstractEventMessageOptions) {
this.id = options.id ?? uuid();
this.eventName = options.eventName;
this.message = options.message ?? options.eventName;
if (typeof options.ts === 'string') {
this.ts = DateTime.fromISO(options.ts) ?? DateTime.now();
} else {
this.ts = options.ts ?? DateTime.now();
}
}
getEventName(): string {
return this.eventName;
}
toString() {
return JSON.stringify(this.serialize());
}
}

View File

@@ -0,0 +1,13 @@
import type { DateTime } from 'luxon';
import { EventMessageTypeNames } from 'n8n-workflow';
import type { AbstractEventPayload } from './AbstractEventPayload';
export interface AbstractEventMessageOptions {
__type?: EventMessageTypeNames;
id?: string;
ts?: DateTime | string;
eventName: string;
message?: string;
payload?: AbstractEventPayload;
anonymize?: boolean;
}

View File

@@ -0,0 +1,5 @@
import type { IWorkflowBase, JsonValue } from 'n8n-workflow';
export interface AbstractEventPayload {
[key: string]: JsonValue | IWorkflowBase | undefined;
}

View File

@@ -0,0 +1,74 @@
import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage';
import { EventMessageTypeNames, JsonObject, JsonValue } from 'n8n-workflow';
import { AbstractEventPayload } from './AbstractEventPayload';
import { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
export const eventNamesAudit = [
'n8n.audit.user.signedup',
'n8n.audit.user.updated',
'n8n.audit.user.deleted',
'n8n.audit.user.invited',
'n8n.audit.user.invitation.accepted',
'n8n.audit.user.reinvited',
'n8n.audit.user.email.failed',
'n8n.audit.user.reset.requested',
'n8n.audit.user.reset',
'n8n.audit.user.credentials.created',
'n8n.audit.user.credentials.shared',
'n8n.audit.user.api.created',
'n8n.audit.user.api.deleted',
'n8n.audit.package.installed',
'n8n.audit.package.updated',
'n8n.audit.package.deleted',
'n8n.audit.workflow.created',
'n8n.audit.workflow.deleted',
'n8n.audit.workflow.updated',
] as const;
export type EventNamesAuditType = typeof eventNamesAudit[number];
// --------------------------------------
// EventMessage class for Audit events
// --------------------------------------
export interface EventPayloadAudit extends AbstractEventPayload {
msg?: JsonValue;
userId?: string;
userEmail?: string;
firstName?: string;
lastName?: string;
}
export interface EventMessageAuditOptions extends AbstractEventMessageOptions {
eventName: EventNamesAuditType;
payload?: EventPayloadAudit;
}
export class EventMessageAudit extends AbstractEventMessage {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
readonly __type = EventMessageTypeNames.audit;
eventName: EventNamesAuditType;
payload: EventPayloadAudit;
constructor(options: EventMessageAuditOptions) {
super(options);
if (options.payload) this.setPayload(options.payload);
if (options.anonymize) {
this.anonymize();
}
}
setPayload(payload: EventPayloadAudit): this {
this.payload = payload;
return this;
}
deserialize(data: JsonObject): this {
if (isEventMessageOptionsWithType(data, this.__type)) {
this.setOptionsOrDefault(data);
if (data.payload) this.setPayload(data.payload as EventPayloadAudit);
}
return this;
}
}

View File

@@ -0,0 +1,39 @@
import { DateTime } from 'luxon';
import { EventMessageTypeNames, JsonObject, JsonValue } from 'n8n-workflow';
export interface EventMessageConfirmSource extends JsonObject {
id: string;
name: string;
}
export class EventMessageConfirm {
readonly __type = EventMessageTypeNames.confirm;
readonly confirm: string;
readonly source?: EventMessageConfirmSource;
readonly ts: DateTime;
constructor(confirm: string, source?: EventMessageConfirmSource) {
this.confirm = confirm;
this.ts = DateTime.now();
if (source) this.source = source;
}
serialize(): JsonValue {
// TODO: filter payload for sensitive info here?
return {
__type: this.__type,
confirm: this.confirm,
ts: this.ts.toISO(),
source: this.source ?? { name: '', id: '' },
};
}
}
export const isEventMessageConfirm = (candidate: unknown): candidate is EventMessageConfirm => {
const o = candidate as EventMessageConfirm;
if (!o) return false;
return o.confirm !== undefined && o.ts !== undefined;
};

View File

@@ -0,0 +1,41 @@
import { EventMessageTypeNames, JsonObject } from 'n8n-workflow';
import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage';
import type { AbstractEventPayload } from './AbstractEventPayload';
import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
export const eventMessageGenericDestinationTestEvent = 'n8n.destination.test';
export interface EventPayloadGeneric extends AbstractEventPayload {
msg?: string;
}
export interface EventMessageGenericOptions extends AbstractEventMessageOptions {
payload?: EventPayloadGeneric;
}
export class EventMessageGeneric extends AbstractEventMessage {
readonly __type = EventMessageTypeNames.generic;
payload: EventPayloadGeneric;
constructor(options: EventMessageGenericOptions) {
super(options);
if (options.payload) this.setPayload(options.payload);
if (options.anonymize) {
this.anonymize();
}
}
setPayload(payload: EventPayloadGeneric): this {
this.payload = payload;
return this;
}
deserialize(data: JsonObject): this {
if (isEventMessageOptionsWithType(data, this.__type)) {
this.setOptionsOrDefault(data);
if (data.payload) this.setPayload(data.payload as EventPayloadGeneric);
}
return this;
}
}

View File

@@ -0,0 +1,49 @@
import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage';
import { EventMessageTypeNames, JsonObject } from 'n8n-workflow';
import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
import type { AbstractEventPayload } from './AbstractEventPayload';
export const eventNamesNode = ['n8n.node.started', 'n8n.node.finished'] as const;
export type EventNamesNodeType = typeof eventNamesNode[number];
// --------------------------------------
// EventMessage class for Node events
// --------------------------------------
export interface EventPayloadNode extends AbstractEventPayload {
msg?: string;
}
export interface EventMessageNodeOptions extends AbstractEventMessageOptions {
eventName: EventNamesNodeType;
payload?: EventPayloadNode | undefined;
}
export class EventMessageNode extends AbstractEventMessage {
readonly __type = EventMessageTypeNames.node;
eventName: EventNamesNodeType;
payload: EventPayloadNode;
constructor(options: EventMessageNodeOptions) {
super(options);
if (options.payload) this.setPayload(options.payload);
if (options.anonymize) {
this.anonymize();
}
}
setPayload(payload: EventPayloadNode): this {
this.payload = payload;
return this;
}
deserialize(data: JsonObject): this {
if (isEventMessageOptionsWithType(data, this.__type)) {
this.setOptionsOrDefault(data);
if (data.payload) this.setPayload(data.payload as EventPayloadNode);
}
return this;
}
}

View File

@@ -0,0 +1,61 @@
import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage';
import { EventMessageTypeNames, IWorkflowBase, JsonObject } from 'n8n-workflow';
import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
import type { AbstractEventPayload } from './AbstractEventPayload';
import { IExecutionBase } from '@/Interfaces';
export const eventNamesWorkflow = [
'n8n.workflow.started',
'n8n.workflow.success',
'n8n.workflow.failed',
] as const;
export type EventNamesWorkflowType = typeof eventNamesWorkflow[number];
// --------------------------------------
// EventMessage class for Workflow events
// --------------------------------------
interface EventPayloadWorkflow extends AbstractEventPayload {
msg?: string;
workflowData?: IWorkflowBase;
executionId?: IExecutionBase['id'];
workflowId?: IWorkflowBase['id'];
}
export interface EventMessageWorkflowOptions extends AbstractEventMessageOptions {
eventName: EventNamesWorkflowType;
payload?: EventPayloadWorkflow | undefined;
}
export class EventMessageWorkflow extends AbstractEventMessage {
readonly __type = EventMessageTypeNames.workflow;
eventName: EventNamesWorkflowType;
payload: EventPayloadWorkflow;
constructor(options: EventMessageWorkflowOptions) {
super(options);
if (options.payload) this.setPayload(options.payload);
if (options.anonymize) {
this.anonymize();
}
}
setPayload(payload: EventPayloadWorkflow): this {
this.payload = payload;
return this;
}
deserialize(data: JsonObject): this {
if (isEventMessageOptionsWithType(data, this.__type)) {
this.setOptionsOrDefault(data);
if (data.payload) this.setPayload(data.payload as EventPayloadWorkflow);
}
return this;
}
}

View File

@@ -0,0 +1,92 @@
import type { EventMessageTypes } from '.';
import { EventMessageGeneric, EventMessageGenericOptions } from './EventMessageGeneric';
import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
import { EventMessageWorkflow, EventMessageWorkflowOptions } from './EventMessageWorkflow';
import { EventMessageTypeNames } from 'n8n-workflow';
export const getEventMessageObjectByType = (
message: AbstractEventMessageOptions,
): EventMessageTypes | null => {
switch (message.__type as EventMessageTypeNames) {
case EventMessageTypeNames.generic:
return new EventMessageGeneric(message as EventMessageGenericOptions);
case EventMessageTypeNames.workflow:
return new EventMessageWorkflow(message as EventMessageWorkflowOptions);
default:
return null;
}
};
interface StringIndexedObject {
[key: string]: StringIndexedObject | string;
}
export function eventGroupFromEventName(eventName: string): string | undefined {
const matches = eventName.match(/^[\w\s]+\.[\w\s]+/);
if (matches && matches?.length > 0) {
return matches[0];
}
return;
}
function dotsToObject2(dottedString: string, o?: StringIndexedObject): StringIndexedObject {
const rootObject: StringIndexedObject = o ?? {};
if (!dottedString) return rootObject;
const parts = dottedString.split('.'); /*?*/
let part: string | undefined;
let obj: StringIndexedObject = rootObject;
while ((part = parts.shift())) {
if (typeof obj[part] !== 'object') {
obj[part] = {
__name: part,
};
}
obj = obj[part] as StringIndexedObject;
}
return rootObject;
}
export function eventListToObject(dottedList: string[]): object {
const result = {};
dottedList.forEach((e) => {
dotsToObject2(e, result);
});
return result;
}
interface StringIndexedChild {
name: string;
children: StringIndexedChild[];
}
export function eventListToObjectTree(dottedList: string[]): StringIndexedChild {
const x: StringIndexedChild = {
name: 'eventTree',
children: [] as unknown as StringIndexedChild[],
};
dottedList.forEach((dottedString: string) => {
const parts = dottedString.split('.');
let part: string | undefined;
let children = x.children;
while ((part = parts.shift())) {
if (part) {
// eslint-disable-next-line @typescript-eslint/no-loop-func
const foundChild = children.find((e) => e.name === part);
if (foundChild) {
children = foundChild.children;
} else {
const newChild: StringIndexedChild = {
name: part,
children: [],
};
children.push(newChild);
children = newChild.children;
}
}
}
});
return x;
}

View File

@@ -0,0 +1,17 @@
import { EventMessageAudit, eventNamesAudit, EventNamesAuditType } from './EventMessageAudit';
import { EventMessageGeneric } from './EventMessageGeneric';
import { EventMessageNode, eventNamesNode, EventNamesNodeType } from './EventMessageNode';
import {
EventMessageWorkflow,
eventNamesWorkflow,
EventNamesWorkflowType,
} from './EventMessageWorkflow';
export type EventNamesTypes = EventNamesAuditType | EventNamesWorkflowType | EventNamesNodeType;
export const eventNamesAll = [...eventNamesAudit, ...eventNamesWorkflow, ...eventNamesNode];
export type EventMessageTypes =
| EventMessageGeneric
| EventMessageWorkflow
| EventMessageAudit
| EventMessageNode;

View File

@@ -0,0 +1,253 @@
import { LoggerProxy, MessageEventBusDestinationOptions } from 'n8n-workflow';
import { DeleteResult } from 'typeorm';
import { EventMessageTypes } from '../EventMessageClasses/';
import type { MessageEventBusDestination } from '../MessageEventBusDestination/MessageEventBusDestination.ee';
import { MessageEventBusLogWriter } from '../MessageEventBusWriter/MessageEventBusLogWriter';
import EventEmitter from 'events';
import config from '@/config';
import * as Db from '@/Db';
import { messageEventBusDestinationFromDb } from '../MessageEventBusDestination/Helpers.ee';
import uniqby from 'lodash.uniqby';
import { EventMessageConfirmSource } from '../EventMessageClasses/EventMessageConfirm';
import {
EventMessageAuditOptions,
EventMessageAudit,
} from '../EventMessageClasses/EventMessageAudit';
import {
EventMessageWorkflowOptions,
EventMessageWorkflow,
} from '../EventMessageClasses/EventMessageWorkflow';
import { isLogStreamingEnabled } from './MessageEventBusHelper';
import { EventMessageNode, EventMessageNodeOptions } from '../EventMessageClasses/EventMessageNode';
import {
EventMessageGeneric,
eventMessageGenericDestinationTestEvent,
} from '../EventMessageClasses/EventMessageGeneric';
export type EventMessageReturnMode = 'sent' | 'unsent' | 'all';
class MessageEventBus extends EventEmitter {
private static instance: MessageEventBus;
isInitialized: boolean;
logWriter: MessageEventBusLogWriter;
destinations: {
[key: string]: MessageEventBusDestination;
} = {};
private pushIntervalTimer: NodeJS.Timer;
constructor() {
super();
this.isInitialized = false;
}
static getInstance(): MessageEventBus {
if (!MessageEventBus.instance) {
MessageEventBus.instance = new MessageEventBus();
}
return MessageEventBus.instance;
}
/**
* Needs to be called once at startup to set the event bus instance up. Will launch the event log writer and,
* if configured to do so, the previously stored event destinations.
*
* Will check for unsent event messages in the previous log files once at startup and try to re-send them.
*
* Sets `isInitialized` to `true` once finished.
*/
async initialize() {
if (this.isInitialized) {
return;
}
LoggerProxy.debug('Initializing event bus...');
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
const savedEventDestinations = await Db.collections.EventDestinations.find({});
if (savedEventDestinations.length > 0) {
for (const destinationData of savedEventDestinations) {
try {
const destination = messageEventBusDestinationFromDb(destinationData);
if (destination) {
await this.addDestination(destination);
}
} catch (error) {
console.log(error);
}
}
}
LoggerProxy.debug('Initializing event writer');
this.logWriter = await MessageEventBusLogWriter.getInstance();
// unsent event check:
// - find unsent messages in current event log(s)
// - cycle event logs and start the logging to a fresh file
// - retry sending events
LoggerProxy.debug('Checking for unsent event messages');
const unsentMessages = await this.getEventsUnsent();
LoggerProxy.debug(
`Start logging into ${
(await this.logWriter?.getThread()?.getLogFileName()) ?? 'unknown filename'
} `,
);
await this.logWriter?.startLogging();
await this.send(unsentMessages);
// if configured, run this test every n ms
if (config.getEnv('eventBus.checkUnsentInterval') > 0) {
if (this.pushIntervalTimer) {
clearInterval(this.pushIntervalTimer);
}
this.pushIntervalTimer = setInterval(async () => {
await this.trySendingUnsent();
}, config.getEnv('eventBus.checkUnsentInterval'));
}
LoggerProxy.debug('MessageEventBus initialized');
this.isInitialized = true;
}
async addDestination(destination: MessageEventBusDestination) {
await this.removeDestination(destination.getId());
this.destinations[destination.getId()] = destination;
this.destinations[destination.getId()].startListening();
return destination;
}
async findDestination(id?: string): Promise<MessageEventBusDestinationOptions[]> {
let result: MessageEventBusDestinationOptions[];
if (id && Object.keys(this.destinations).includes(id)) {
result = [this.destinations[id].serialize()];
} else {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
result = Object.keys(this.destinations).map((e) => this.destinations[e].serialize());
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
return result.sort((a, b) => (a.__type ?? '').localeCompare(b.__type ?? ''));
}
async removeDestination(id: string): Promise<DeleteResult | undefined> {
let result;
if (Object.keys(this.destinations).includes(id)) {
await this.destinations[id].close();
result = await this.destinations[id].deleteFromDb();
delete this.destinations[id];
}
return result;
}
private async trySendingUnsent(msgs?: EventMessageTypes[]) {
const unsentMessages = msgs ?? (await this.getEventsUnsent());
if (unsentMessages.length > 0) {
LoggerProxy.debug(`Found unsent event messages: ${unsentMessages.length}`);
for (const unsentMsg of unsentMessages) {
LoggerProxy.debug(`Retrying: ${unsentMsg.id} ${unsentMsg.__type}`);
await this.emitMessage(unsentMsg);
}
}
}
async close() {
LoggerProxy.debug('Shutting down event writer...');
await this.logWriter?.close();
for (const destinationName of Object.keys(this.destinations)) {
LoggerProxy.debug(
`Shutting down event destination ${this.destinations[destinationName].getId()}...`,
);
await this.destinations[destinationName].close();
}
LoggerProxy.debug('EventBus shut down.');
}
async send(msgs: EventMessageTypes | EventMessageTypes[]) {
if (!Array.isArray(msgs)) {
msgs = [msgs];
}
for (const msg of msgs) {
await this.logWriter?.putMessage(msg);
await this.emitMessage(msg);
}
}
async testDestination(destinationId: string): Promise<boolean> {
const testMessage = new EventMessageGeneric({
eventName: eventMessageGenericDestinationTestEvent,
});
const destination = await this.findDestination(destinationId);
if (destination.length > 0) {
const sendResult = await this.destinations[destinationId].receiveFromEventBus(testMessage);
return sendResult;
}
return false;
}
async confirmSent(msg: EventMessageTypes, source?: EventMessageConfirmSource) {
await this.logWriter?.confirmMessageSent(msg.id, source);
}
private async emitMessage(msg: EventMessageTypes) {
// generic emit for external modules to capture events
// this is for internal use ONLY and not for use with custom destinations!
this.emit('message', msg);
LoggerProxy.debug(`Listeners: ${this.eventNames().join(',')}`);
// if there are no set up destinations, immediately mark the event as sent
if (!isLogStreamingEnabled() || Object.keys(this.destinations).length === 0) {
await this.confirmSent(msg, { id: '0', name: 'eventBus' });
} else {
for (const destinationName of Object.keys(this.destinations)) {
this.emit(this.destinations[destinationName].getId(), msg);
}
}
}
async getEvents(mode: EventMessageReturnMode = 'all'): Promise<EventMessageTypes[]> {
let queryResult: EventMessageTypes[];
switch (mode) {
case 'all':
queryResult = await this.logWriter?.getMessages();
break;
case 'sent':
queryResult = await this.logWriter?.getMessagesSent();
break;
case 'unsent':
queryResult = await this.logWriter?.getMessagesUnsent();
}
const filtered = uniqby(queryResult, 'id');
return filtered;
}
async getEventsSent(): Promise<EventMessageTypes[]> {
const sentMessages = await this.getEvents('sent');
return sentMessages;
}
async getEventsUnsent(): Promise<EventMessageTypes[]> {
const unSentMessages = await this.getEvents('unsent');
return unSentMessages;
}
/**
* Convenience Methods
*/
async sendAuditEvent(options: EventMessageAuditOptions) {
await this.send(new EventMessageAudit(options));
}
async sendWorkflowEvent(options: EventMessageWorkflowOptions) {
await this.send(new EventMessageWorkflow(options));
}
async sendNodeEvent(options: EventMessageNodeOptions) {
await this.send(new EventMessageNode(options));
}
}
export const eventBus = MessageEventBus.getInstance();

View File

@@ -0,0 +1,7 @@
import config from '@/config';
import { getLicense } from '@/License';
export function isLogStreamingEnabled(): boolean {
const license = getLicense();
return config.getEnv('enterprise.features.logStreaming') || license.isLogStreamingEnabled();
}

View File

@@ -0,0 +1,28 @@
/* eslint-disable import/no-cycle */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
import { MessageEventBusDestinationTypeNames } from 'n8n-workflow';
import type { EventDestinations } from '@/databases/entities/MessageEventBusDestinationEntity';
import type { MessageEventBusDestination } from './MessageEventBusDestination.ee';
import { MessageEventBusDestinationSentry } from './MessageEventBusDestinationSentry.ee';
import { MessageEventBusDestinationSyslog } from './MessageEventBusDestinationSyslog.ee';
import { MessageEventBusDestinationWebhook } from './MessageEventBusDestinationWebhook.ee';
export function messageEventBusDestinationFromDb(
dbData: EventDestinations,
): MessageEventBusDestination | null {
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-assignment
const destinationData = dbData.destination;
if ('__type' in destinationData) {
switch (destinationData.__type) {
case MessageEventBusDestinationTypeNames.sentry:
return MessageEventBusDestinationSentry.deserialize(destinationData);
case MessageEventBusDestinationTypeNames.syslog:
return MessageEventBusDestinationSyslog.deserialize(destinationData);
case MessageEventBusDestinationTypeNames.webhook:
return MessageEventBusDestinationWebhook.deserialize(destinationData);
default:
console.log('MessageEventBusDestination __type unknown');
}
}
return null;
}

View File

@@ -0,0 +1,125 @@
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { v4 as uuid } from 'uuid';
import {
INodeCredentials,
LoggerProxy,
MessageEventBusDestinationOptions,
MessageEventBusDestinationTypeNames,
} from 'n8n-workflow';
import * as Db from '@/Db';
import { AbstractEventMessage } from '../EventMessageClasses/AbstractEventMessage';
import { EventMessageTypes } from '../EventMessageClasses';
import { eventBus } from '..';
import { DeleteResult, InsertResult } from 'typeorm';
export abstract class MessageEventBusDestination implements MessageEventBusDestinationOptions {
// Since you can't have static abstract functions - this just serves as a reminder that you need to implement these. Please.
// static abstract deserialize(): MessageEventBusDestination | null;
readonly id: string;
__type: MessageEventBusDestinationTypeNames;
label: string;
enabled: boolean;
subscribedEvents: string[];
credentials: INodeCredentials = {};
anonymizeAuditMessages: boolean;
constructor(options: MessageEventBusDestinationOptions) {
this.id = !options.id || options.id.length !== 36 ? uuid() : options.id;
this.__type = options.__type ?? MessageEventBusDestinationTypeNames.abstract;
this.label = options.label ?? 'Log Destination';
this.enabled = options.enabled ?? false;
this.subscribedEvents = options.subscribedEvents ?? [];
this.anonymizeAuditMessages = options.anonymizeAuditMessages ?? false;
if (options.credentials) this.credentials = options.credentials;
LoggerProxy.debug(`${this.__type}(${this.id}) event destination constructed`);
}
startListening() {
if (this.enabled) {
eventBus.on(this.getId(), async (msg: EventMessageTypes) => {
await this.receiveFromEventBus(msg);
});
LoggerProxy.debug(`${this.id} listener started`);
}
}
stopListening() {
eventBus.removeAllListeners(this.getId());
}
enable() {
this.enabled = true;
this.startListening();
}
disable() {
this.enabled = false;
this.stopListening();
}
getId() {
return this.id;
}
hasSubscribedToEvent(msg: AbstractEventMessage) {
if (!this.enabled) return false;
for (const eventName of this.subscribedEvents) {
if (eventName === '*' || msg.eventName.startsWith(eventName)) {
return true;
}
}
return false;
}
async saveToDb() {
const data = {
id: this.getId(),
destination: this.serialize(),
};
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
const dbResult: InsertResult = await Db.collections.EventDestinations.upsert(data, {
skipUpdateIfNoValuesChanged: true,
conflictPaths: ['id'],
});
Db.collections.EventDestinations.createQueryBuilder().insert().into('something').onConflict('');
return dbResult;
}
async deleteFromDb() {
return MessageEventBusDestination.deleteFromDb(this.getId());
}
static async deleteFromDb(id: string): Promise<DeleteResult> {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
const dbResult = await Db.collections.EventDestinations.delete({ id });
return dbResult;
}
serialize(): MessageEventBusDestinationOptions {
return {
__type: this.__type,
id: this.getId(),
label: this.label,
enabled: this.enabled,
subscribedEvents: this.subscribedEvents,
anonymizeAuditMessages: this.anonymizeAuditMessages,
};
}
abstract receiveFromEventBus(msg: AbstractEventMessage): Promise<boolean>;
toString() {
return JSON.stringify(this.serialize());
}
close(): void | Promise<void> {
this.stopListening();
}
}

View File

@@ -0,0 +1,137 @@
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-return */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { MessageEventBusDestination } from './MessageEventBusDestination.ee';
import * as Sentry from '@sentry/node';
import { eventBus } from '../MessageEventBus/MessageEventBus';
import {
LoggerProxy,
MessageEventBusDestinationOptions,
MessageEventBusDestinationSentryOptions,
MessageEventBusDestinationTypeNames,
} from 'n8n-workflow';
import { GenericHelpers } from '../..';
import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper';
import { EventMessageTypes } from '../EventMessageClasses';
import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric';
export const isMessageEventBusDestinationSentryOptions = (
candidate: unknown,
): candidate is MessageEventBusDestinationSentryOptions => {
const o = candidate as MessageEventBusDestinationSentryOptions;
if (!o) return false;
return o.dsn !== undefined;
};
export class MessageEventBusDestinationSentry
extends MessageEventBusDestination
implements MessageEventBusDestinationSentryOptions
{
dsn: string;
tracesSampleRate = 1.0;
sendPayload: boolean;
sentryClient?: Sentry.NodeClient;
constructor(options: MessageEventBusDestinationSentryOptions) {
super(options);
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
this.label = options.label ?? 'Sentry DSN';
this.__type = options.__type ?? MessageEventBusDestinationTypeNames.sentry;
this.dsn = options.dsn;
if (options.sendPayload) this.sendPayload = options.sendPayload;
if (options.tracesSampleRate) this.tracesSampleRate = options.tracesSampleRate;
const { ENVIRONMENT: environment } = process.env;
GenericHelpers.getVersions()
.then((versions) => {
this.sentryClient = new Sentry.NodeClient({
dsn: this.dsn,
tracesSampleRate: this.tracesSampleRate,
environment,
release: versions.cli,
transport: Sentry.makeNodeTransport,
integrations: Sentry.defaultIntegrations,
stackParser: Sentry.defaultStackParser,
});
LoggerProxy.debug(`MessageEventBusDestinationSentry with id ${this.getId()} initialized`);
})
.catch((error) => {
console.error(error);
});
}
async receiveFromEventBus(msg: EventMessageTypes): Promise<boolean> {
let sendResult = false;
if (!this.sentryClient) return sendResult;
if (msg.eventName !== eventMessageGenericDestinationTestEvent) {
if (!isLogStreamingEnabled()) return sendResult;
if (!this.hasSubscribedToEvent(msg)) return sendResult;
}
try {
const payload = this.anonymizeAuditMessages ? msg.anonymize() : msg.payload;
const scope: Sentry.Scope = new Sentry.Scope();
const level = (
msg.eventName.toLowerCase().endsWith('error') ? 'error' : 'log'
) as Sentry.SeverityLevel;
scope.setLevel(level);
scope.setTags({
event: msg.getEventName(),
logger: this.label ?? this.getId(),
app: 'n8n',
});
if (this.sendPayload) {
scope.setExtras(payload);
}
const sentryResult = this.sentryClient.captureMessage(
msg.message ?? msg.eventName,
level,
{ event_id: msg.id, data: payload },
scope,
);
if (sentryResult) {
await eventBus.confirmSent(msg, { id: this.id, name: this.label });
sendResult = true;
}
} catch (error) {
console.log(error);
}
return sendResult;
}
serialize(): MessageEventBusDestinationSentryOptions {
const abstractSerialized = super.serialize();
return {
...abstractSerialized,
dsn: this.dsn,
tracesSampleRate: this.tracesSampleRate,
sendPayload: this.sendPayload,
};
}
static deserialize(
data: MessageEventBusDestinationOptions,
): MessageEventBusDestinationSentry | null {
if (
'__type' in data &&
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
data.__type === MessageEventBusDestinationTypeNames.sentry &&
isMessageEventBusDestinationSentryOptions(data)
) {
return new MessageEventBusDestinationSentry(data);
}
return null;
}
toString() {
return JSON.stringify(this.serialize());
}
async close() {
await super.close();
await this.sentryClient?.close();
}
}

View File

@@ -0,0 +1,149 @@
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/no-non-null-assertion */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
import syslog from 'syslog-client';
import { eventBus } from '../MessageEventBus/MessageEventBus';
import {
LoggerProxy,
MessageEventBusDestinationOptions,
MessageEventBusDestinationSyslogOptions,
MessageEventBusDestinationTypeNames,
} from 'n8n-workflow';
import { MessageEventBusDestination } from './MessageEventBusDestination.ee';
import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper';
import { EventMessageTypes } from '../EventMessageClasses';
import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric';
export const isMessageEventBusDestinationSyslogOptions = (
candidate: unknown,
): candidate is MessageEventBusDestinationSyslogOptions => {
const o = candidate as MessageEventBusDestinationSyslogOptions;
if (!o) return false;
return o.host !== undefined;
};
export class MessageEventBusDestinationSyslog
extends MessageEventBusDestination
implements MessageEventBusDestinationSyslogOptions
{
client: syslog.Client;
expectedStatusCode?: number;
host: string;
port: number;
protocol: 'udp' | 'tcp';
facility: syslog.Facility;
app_name: string;
eol: string;
constructor(options: MessageEventBusDestinationSyslogOptions) {
super(options);
this.__type = options.__type ?? MessageEventBusDestinationTypeNames.syslog;
this.label = options.label ?? 'Syslog Server';
this.host = options.host ?? 'localhost';
this.port = options.port ?? 514;
this.protocol = options.protocol ?? 'udp';
this.facility = options.facility ?? syslog.Facility.Local0;
this.app_name = options.app_name ?? 'n8n';
this.eol = options.eol ?? '\n';
this.expectedStatusCode = options.expectedStatusCode ?? 200;
this.client = syslog.createClient(this.host, {
appName: this.app_name,
facility: syslog.Facility.Local0,
// severity: syslog.Severity.Error,
port: this.port,
transport:
options.protocol !== undefined && options.protocol === 'tcp'
? syslog.Transport.Tcp
: syslog.Transport.Udp,
});
LoggerProxy.debug(`MessageEventBusDestinationSyslog with id ${this.getId()} initialized`);
this.client.on('error', function (error) {
console.error(error);
});
}
async receiveFromEventBus(msg: EventMessageTypes): Promise<boolean> {
let sendResult = false;
if (msg.eventName !== eventMessageGenericDestinationTestEvent) {
if (!isLogStreamingEnabled()) return sendResult;
if (!this.hasSubscribedToEvent(msg)) return sendResult;
}
try {
const serializedMessage = msg.serialize();
if (this.anonymizeAuditMessages) {
serializedMessage.payload = msg.anonymize();
}
delete serializedMessage.__type;
this.client.log(
JSON.stringify(serializedMessage),
{
severity: msg.eventName.toLowerCase().endsWith('error')
? syslog.Severity.Error
: syslog.Severity.Debug,
msgid: msg.id,
timestamp: msg.ts.toJSDate(),
},
async (error) => {
if (error) {
console.log(error);
} else {
await eventBus.confirmSent(msg, { id: this.id, name: this.label });
sendResult = true;
}
},
);
} catch (error) {
console.log(error);
}
if (msg.eventName === eventMessageGenericDestinationTestEvent) {
await new Promise((resolve) => setTimeout(resolve, 500));
}
return sendResult;
}
serialize(): MessageEventBusDestinationSyslogOptions {
const abstractSerialized = super.serialize();
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return {
...abstractSerialized,
expectedStatusCode: this.expectedStatusCode,
host: this.host,
port: this.port,
protocol: this.protocol,
facility: this.facility,
app_name: this.app_name,
eol: this.eol,
};
}
static deserialize(
data: MessageEventBusDestinationOptions,
): MessageEventBusDestinationSyslog | null {
if (
'__type' in data &&
data.__type === MessageEventBusDestinationTypeNames.syslog &&
isMessageEventBusDestinationSyslogOptions(data)
) {
return new MessageEventBusDestinationSyslog(data);
}
return null;
}
toString() {
return JSON.stringify(this.serialize());
}
async close() {
await super.close();
this.client.close();
}
}

View File

@@ -0,0 +1,372 @@
/* eslint-disable import/no-cycle */
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-return */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/no-unused-vars */
/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
import { MessageEventBusDestination } from './MessageEventBusDestination.ee';
import axios, { AxiosRequestConfig, Method } from 'axios';
import { eventBus } from '../MessageEventBus/MessageEventBus';
import { EventMessageTypes } from '../EventMessageClasses';
import {
jsonParse,
LoggerProxy,
MessageEventBusDestinationOptions,
MessageEventBusDestinationTypeNames,
MessageEventBusDestinationWebhookOptions,
MessageEventBusDestinationWebhookParameterItem,
MessageEventBusDestinationWebhookParameterOptions,
} from 'n8n-workflow';
import { CredentialsHelper } from '../../CredentialsHelper';
import { UserSettings } from 'n8n-core';
import { Agent as HTTPSAgent } from 'https';
import config from '../../config';
import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper';
import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric';
export const isMessageEventBusDestinationWebhookOptions = (
candidate: unknown,
): candidate is MessageEventBusDestinationWebhookOptions => {
const o = candidate as MessageEventBusDestinationWebhookOptions;
if (!o) return false;
return o.url !== undefined;
};
export class MessageEventBusDestinationWebhook
extends MessageEventBusDestination
implements MessageEventBusDestinationWebhookOptions
{
url: string;
responseCodeMustMatch = false;
expectedStatusCode = 200;
method = 'POST';
authentication: 'predefinedCredentialType' | 'genericCredentialType' | 'none' = 'none';
sendQuery = false;
sendHeaders = false;
genericAuthType = '';
nodeCredentialType = '';
specifyHeaders = '';
specifyQuery = '';
jsonQuery = '';
jsonHeaders = '';
headerParameters: MessageEventBusDestinationWebhookParameterItem = { parameters: [] };
queryParameters: MessageEventBusDestinationWebhookParameterItem = { parameters: [] };
options: MessageEventBusDestinationWebhookParameterOptions = {};
sendPayload = true;
credentialsHelper?: CredentialsHelper;
axiosRequestOptions: AxiosRequestConfig;
constructor(options: MessageEventBusDestinationWebhookOptions) {
super(options);
this.url = options.url;
this.label = options.label ?? 'Webhook Endpoint';
this.__type = options.__type ?? MessageEventBusDestinationTypeNames.webhook;
if (options.responseCodeMustMatch) this.responseCodeMustMatch = options.responseCodeMustMatch;
if (options.expectedStatusCode) this.expectedStatusCode = options.expectedStatusCode;
if (options.method) this.method = options.method;
if (options.authentication) this.authentication = options.authentication;
if (options.sendQuery) this.sendQuery = options.sendQuery;
if (options.sendHeaders) this.sendHeaders = options.sendHeaders;
if (options.genericAuthType) this.genericAuthType = options.genericAuthType;
if (options.nodeCredentialType) this.nodeCredentialType = options.nodeCredentialType;
if (options.specifyHeaders) this.specifyHeaders = options.specifyHeaders;
if (options.specifyQuery) this.specifyQuery = options.specifyQuery;
if (options.jsonQuery) this.jsonQuery = options.jsonQuery;
if (options.jsonHeaders) this.jsonHeaders = options.jsonHeaders;
if (options.headerParameters) this.headerParameters = options.headerParameters;
if (options.queryParameters) this.queryParameters = options.queryParameters;
if (options.sendPayload) this.sendPayload = options.sendPayload;
if (options.options) this.options = options.options;
LoggerProxy.debug(`MessageEventBusDestinationWebhook with id ${this.getId()} initialized`);
}
async matchDecryptedCredentialType(credentialType: string) {
const foundCredential = Object.entries(this.credentials).find((e) => e[0] === credentialType);
if (foundCredential) {
const timezone = config.getEnv('generic.timezone');
const credentialsDecrypted = await this.credentialsHelper?.getDecrypted(
foundCredential[1],
foundCredential[0],
'internal',
timezone,
true,
);
return credentialsDecrypted;
}
return null;
}
async generateAxiosOptions() {
if (this.axiosRequestOptions?.url) {
return;
}
this.axiosRequestOptions = {
headers: {},
method: this.method as Method,
url: this.url,
maxRedirects: 0,
} as AxiosRequestConfig;
if (this.credentialsHelper === undefined) {
let encryptionKey: string | undefined;
try {
encryptionKey = await UserSettings.getEncryptionKey();
} catch (_) {}
if (encryptionKey) {
this.credentialsHelper = new CredentialsHelper(encryptionKey);
}
}
const sendQuery = this.sendQuery;
const specifyQuery = this.specifyQuery;
const sendPayload = this.sendPayload;
const sendHeaders = this.sendHeaders;
const specifyHeaders = this.specifyHeaders;
if (this.options.allowUnauthorizedCerts) {
this.axiosRequestOptions.httpsAgent = new HTTPSAgent({ rejectUnauthorized: false });
}
if (this.options.redirect?.followRedirects) {
this.axiosRequestOptions.maxRedirects = this.options.redirect?.maxRedirects;
}
if (this.options.proxy) {
this.axiosRequestOptions.proxy = this.options.proxy;
}
if (this.options.timeout) {
this.axiosRequestOptions.timeout = this.options.timeout;
} else {
this.axiosRequestOptions.timeout = 10000;
}
if (this.sendQuery && this.options.queryParameterArrays) {
Object.assign(this.axiosRequestOptions, {
qsStringifyOptions: { arrayFormat: this.options.queryParameterArrays },
});
}
const parametersToKeyValue = async (
// eslint-disable-next-line @typescript-eslint/no-explicit-any
acc: Promise<{ [key: string]: any }>,
cur: { name: string; value: string; parameterType?: string; inputDataFieldName?: string },
) => {
const acumulator = await acc;
acumulator[cur.name] = cur.value;
return acumulator;
};
// Get parameters defined in the UI
if (sendQuery && this.queryParameters.parameters) {
if (specifyQuery === 'keypair') {
this.axiosRequestOptions.params = this.queryParameters.parameters.reduce(
parametersToKeyValue,
Promise.resolve({}),
);
} else if (specifyQuery === 'json') {
// query is specified using JSON
try {
JSON.parse(this.jsonQuery);
} catch (_) {
console.log('JSON parameter need to be an valid JSON');
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
this.axiosRequestOptions.params = jsonParse(this.jsonQuery);
}
}
// Get parameters defined in the UI
if (sendHeaders && this.headerParameters.parameters) {
if (specifyHeaders === 'keypair') {
this.axiosRequestOptions.headers = await this.headerParameters.parameters.reduce(
parametersToKeyValue,
Promise.resolve({}),
);
} else if (specifyHeaders === 'json') {
// body is specified using JSON
try {
JSON.parse(this.jsonHeaders);
} catch (_) {
console.log('JSON parameter need to be an valid JSON');
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
this.axiosRequestOptions.headers = jsonParse(this.jsonHeaders);
}
}
// default for bodyContentType.raw
if (this.axiosRequestOptions.headers === undefined) {
this.axiosRequestOptions.headers = {};
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
this.axiosRequestOptions.headers['Content-Type'] = 'application/json';
}
serialize(): MessageEventBusDestinationWebhookOptions {
const abstractSerialized = super.serialize();
return {
...abstractSerialized,
url: this.url,
responseCodeMustMatch: this.responseCodeMustMatch,
expectedStatusCode: this.expectedStatusCode,
method: this.method,
authentication: this.authentication,
sendQuery: this.sendQuery,
sendHeaders: this.sendHeaders,
genericAuthType: this.genericAuthType,
nodeCredentialType: this.nodeCredentialType,
specifyHeaders: this.specifyHeaders,
specifyQuery: this.specifyQuery,
jsonQuery: this.jsonQuery,
jsonHeaders: this.jsonHeaders,
headerParameters: this.headerParameters,
queryParameters: this.queryParameters,
sendPayload: this.sendPayload,
options: this.options,
credentials: this.credentials,
};
}
static deserialize(
data: MessageEventBusDestinationOptions,
): MessageEventBusDestinationWebhook | null {
if (
'__type' in data &&
data.__type === MessageEventBusDestinationTypeNames.webhook &&
isMessageEventBusDestinationWebhookOptions(data)
) {
return new MessageEventBusDestinationWebhook(data);
}
return null;
}
async receiveFromEventBus(msg: EventMessageTypes): Promise<boolean> {
let sendResult = false;
if (msg.eventName !== eventMessageGenericDestinationTestEvent) {
if (!isLogStreamingEnabled()) return sendResult;
if (!this.hasSubscribedToEvent(msg)) return sendResult;
}
// at first run, build this.requestOptions with the destination settings
await this.generateAxiosOptions();
const payload = this.anonymizeAuditMessages ? msg.anonymize() : msg.payload;
if (['PATCH', 'POST', 'PUT', 'GET'].includes(this.method.toUpperCase())) {
if (this.sendPayload) {
this.axiosRequestOptions.data = {
...msg,
__type: undefined,
payload,
ts: msg.ts.toISO(),
};
} else {
this.axiosRequestOptions.data = {
...msg,
__type: undefined,
payload: undefined,
ts: msg.ts.toISO(),
};
}
}
// TODO: implement extra auth requests
let httpBasicAuth;
let httpDigestAuth;
let httpHeaderAuth;
let httpQueryAuth;
let oAuth1Api;
let oAuth2Api;
if (this.authentication === 'genericCredentialType') {
if (this.genericAuthType === 'httpBasicAuth') {
try {
httpBasicAuth = await this.matchDecryptedCredentialType('httpBasicAuth');
} catch (_) {}
} else if (this.genericAuthType === 'httpDigestAuth') {
try {
httpDigestAuth = await this.matchDecryptedCredentialType('httpDigestAuth');
} catch (_) {}
} else if (this.genericAuthType === 'httpHeaderAuth') {
try {
httpHeaderAuth = await this.matchDecryptedCredentialType('httpHeaderAuth');
} catch (_) {}
} else if (this.genericAuthType === 'httpQueryAuth') {
try {
httpQueryAuth = await this.matchDecryptedCredentialType('httpQueryAuth');
} catch (_) {}
} else if (this.genericAuthType === 'oAuth1Api') {
try {
oAuth1Api = await this.matchDecryptedCredentialType('oAuth1Api');
} catch (_) {}
} else if (this.genericAuthType === 'oAuth2Api') {
try {
oAuth2Api = await this.matchDecryptedCredentialType('oAuth2Api');
} catch (_) {}
}
}
if (httpBasicAuth) {
// Add credentials if any are set
this.axiosRequestOptions.auth = {
username: httpBasicAuth.user as string,
password: httpBasicAuth.password as string,
};
} else if (httpHeaderAuth) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
this.axiosRequestOptions.headers[httpHeaderAuth.name as string] = httpHeaderAuth.value;
} else if (httpQueryAuth) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
this.axiosRequestOptions.params[httpQueryAuth.name as string] = httpQueryAuth.value;
} else if (httpDigestAuth) {
this.axiosRequestOptions.auth = {
username: httpDigestAuth.user as string,
password: httpDigestAuth.password as string,
};
}
try {
const requestResponse = await axios.request(this.axiosRequestOptions);
if (requestResponse) {
if (this.responseCodeMustMatch) {
if (requestResponse.status === this.expectedStatusCode) {
await eventBus.confirmSent(msg, { id: this.id, name: this.label });
sendResult = true;
} else {
sendResult = false;
}
} else {
await eventBus.confirmSent(msg, { id: this.id, name: this.label });
sendResult = true;
}
}
} catch (error) {
console.error(error);
}
return sendResult;
}
}

View File

@@ -0,0 +1,221 @@
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-explicit-any */
import { isEventMessageOptions } from '../EventMessageClasses/AbstractEventMessage';
import { UserSettings } from 'n8n-core';
import path, { parse } from 'path';
import { ModuleThread, spawn, Thread, Worker } from 'threads';
import { MessageEventBusLogWriterWorker } from './MessageEventBusLogWriterWorker';
import { createReadStream, existsSync } from 'fs';
import readline from 'readline';
import { jsonParse, LoggerProxy } from 'n8n-workflow';
import remove from 'lodash.remove';
import config from '@/config';
import { getEventMessageObjectByType } from '../EventMessageClasses/Helpers';
import type { EventMessageReturnMode } from '../MessageEventBus/MessageEventBus';
import type { EventMessageTypes } from '../EventMessageClasses';
import {
EventMessageConfirm,
EventMessageConfirmSource,
isEventMessageConfirm,
} from '../EventMessageClasses/EventMessageConfirm';
import { once as eventOnce } from 'events';
interface MessageEventBusLogWriterOptions {
syncFileAccess?: boolean;
logBaseName?: string;
logBasePath?: string;
keepLogCount?: number;
maxFileSizeInKB?: number;
}
/**
* MessageEventBusWriter for Files
*/
export class MessageEventBusLogWriter {
private static instance: MessageEventBusLogWriter;
static options: Required<MessageEventBusLogWriterOptions>;
private worker: ModuleThread<MessageEventBusLogWriterWorker> | null;
/**
* Instantiates the Writer and the corresponding worker thread.
* To actually start logging, call startLogging() function on the instance.
*
* **Note** that starting to log will archive existing logs, so handle unsent events first before calling startLogging()
*/
static async getInstance(
options?: MessageEventBusLogWriterOptions,
): Promise<MessageEventBusLogWriter> {
if (!MessageEventBusLogWriter.instance) {
MessageEventBusLogWriter.instance = new MessageEventBusLogWriter();
MessageEventBusLogWriter.options = {
logBaseName: options?.logBaseName ?? config.getEnv('eventBus.logWriter.logBaseName'),
logBasePath: options?.logBasePath ?? UserSettings.getUserN8nFolderPath(),
syncFileAccess:
options?.syncFileAccess ?? config.getEnv('eventBus.logWriter.syncFileAccess'),
keepLogCount: options?.keepLogCount ?? config.getEnv('eventBus.logWriter.keepLogCount'),
maxFileSizeInKB:
options?.maxFileSizeInKB ?? config.getEnv('eventBus.logWriter.maxFileSizeInKB'),
};
await MessageEventBusLogWriter.instance.startThread();
}
return MessageEventBusLogWriter.instance;
}
/**
* First archives existing log files one history level upwards,
* then starts logging events into a fresh event log
*/
async startLogging() {
await MessageEventBusLogWriter.instance.getThread()?.startLogging();
}
/**
* Pauses all logging. Events are still received by the worker, they just are not logged any more
*/
async pauseLogging() {
await MessageEventBusLogWriter.instance.getThread()?.pauseLogging();
}
private async startThread() {
if (this.worker) {
await this.close();
}
await MessageEventBusLogWriter.instance.spawnThread();
await MessageEventBusLogWriter.instance
.getThread()
?.initialize(
path.join(
MessageEventBusLogWriter.options.logBasePath,
MessageEventBusLogWriter.options.logBaseName,
),
MessageEventBusLogWriter.options.syncFileAccess,
MessageEventBusLogWriter.options.keepLogCount,
MessageEventBusLogWriter.options.maxFileSizeInKB,
);
}
private async spawnThread(): Promise<boolean> {
this.worker = await spawn<MessageEventBusLogWriterWorker>(
new Worker(`${parse(__filename).name}Worker`),
);
if (this.worker) {
Thread.errors(this.worker).subscribe(async (error) => {
LoggerProxy.error('Event Bus Log Writer thread error', error);
await MessageEventBusLogWriter.instance.startThread();
});
return true;
}
return false;
}
getThread(): ModuleThread<MessageEventBusLogWriterWorker> | undefined {
if (this.worker) {
return this.worker;
}
return;
}
async close(): Promise<void> {
if (this.worker) {
await Thread.terminate(this.worker);
this.worker = null;
}
}
async putMessage(msg: EventMessageTypes): Promise<void> {
if (this.worker) {
await this.worker.appendMessageToLog(msg.serialize());
}
}
async confirmMessageSent(msgId: string, source?: EventMessageConfirmSource): Promise<void> {
if (this.worker) {
await this.worker.confirmMessageSent(new EventMessageConfirm(msgId, source).serialize());
}
}
async getMessages(
mode: EventMessageReturnMode = 'all',
includePreviousLog = true,
): Promise<EventMessageTypes[]> {
const logFileName0 = await MessageEventBusLogWriter.instance.getThread()?.getLogFileName();
const logFileName1 = includePreviousLog
? await MessageEventBusLogWriter.instance.getThread()?.getLogFileName(1)
: undefined;
const results: {
loggedMessages: EventMessageTypes[];
sentMessages: EventMessageTypes[];
} = {
loggedMessages: [],
sentMessages: [],
};
if (logFileName0) {
await this.readLoggedMessagesFromFile(results, mode, logFileName0);
}
if (logFileName1) {
await this.readLoggedMessagesFromFile(results, mode, logFileName1);
}
switch (mode) {
case 'all':
case 'unsent':
return results.loggedMessages;
case 'sent':
return results.sentMessages;
}
}
async readLoggedMessagesFromFile(
results: {
loggedMessages: EventMessageTypes[];
sentMessages: EventMessageTypes[];
},
mode: EventMessageReturnMode,
logFileName: string,
): Promise<{
loggedMessages: EventMessageTypes[];
sentMessages: EventMessageTypes[];
}> {
if (logFileName && existsSync(logFileName)) {
try {
const rl = readline.createInterface({
input: createReadStream(logFileName),
crlfDelay: Infinity,
});
rl.on('line', (line) => {
try {
const json = jsonParse(line);
if (isEventMessageOptions(json) && json.__type !== undefined) {
const msg = getEventMessageObjectByType(json);
if (msg !== null) results.loggedMessages.push(msg);
}
if (isEventMessageConfirm(json) && mode !== 'all') {
const removedMessage = remove(results.loggedMessages, (e) => e.id === json.confirm);
if (mode === 'sent') {
results.sentMessages.push(...removedMessage);
}
}
} catch {
LoggerProxy.error(
`Error reading line messages from file: ${logFileName}, line: ${line}`,
);
}
});
// wait for stream to finish before continue
await eventOnce(rl, 'close');
} catch {
LoggerProxy.error(`Error reading logged messages from file: ${logFileName}`);
}
}
return results;
}
async getMessagesSent(): Promise<EventMessageTypes[]> {
return this.getMessages('sent');
}
async getMessagesUnsent(): Promise<EventMessageTypes[]> {
return this.getMessages('unsent');
}
}

View File

@@ -0,0 +1,145 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { appendFileSync, existsSync, rmSync, renameSync, openSync, closeSync } from 'fs';
import { appendFile, stat } from 'fs/promises';
import { expose, isWorkerRuntime } from 'threads/worker';
// -----------------------------------------
// * This part runs in the Worker Thread ! *
// -----------------------------------------
// all references to and imports from classes have been remove to keep memory usage low
let logFileBasePath = '';
let loggingPaused = true;
let syncFileAccess = false;
let keepFiles = 10;
let fileStatTimer: NodeJS.Timer;
let maxLogFileSizeInKB = 102400;
function setLogFileBasePath(basePath: string) {
logFileBasePath = basePath;
}
function setUseSyncFileAccess(useSync: boolean) {
syncFileAccess = useSync;
}
function setMaxLogFileSizeInKB(maxSizeInKB: number) {
maxLogFileSizeInKB = maxSizeInKB;
}
function setKeepFiles(keepNumberOfFiles: number) {
if (keepNumberOfFiles < 1) {
keepNumberOfFiles = 1;
}
keepFiles = keepNumberOfFiles;
}
function buildLogFileNameWithCounter(counter?: number): string {
if (counter) {
return `${logFileBasePath}-${counter}.log`;
} else {
return `${logFileBasePath}.log`;
}
}
function cleanAllLogs() {
for (let i = 0; i <= keepFiles; i++) {
if (existsSync(buildLogFileNameWithCounter(i))) {
rmSync(buildLogFileNameWithCounter(i));
}
}
}
/**
* Runs synchronously and cycles through log files up to the max amount kept
*/
function renameAndCreateLogs() {
if (existsSync(buildLogFileNameWithCounter(keepFiles))) {
rmSync(buildLogFileNameWithCounter(keepFiles));
}
for (let i = keepFiles - 1; i >= 0; i--) {
if (existsSync(buildLogFileNameWithCounter(i))) {
renameSync(buildLogFileNameWithCounter(i), buildLogFileNameWithCounter(i + 1));
}
}
const f = openSync(buildLogFileNameWithCounter(), 'a');
closeSync(f);
}
async function checkFileSize(path: string) {
const fileStat = await stat(path);
if (fileStat.size / 1024 > maxLogFileSizeInKB) {
renameAndCreateLogs();
}
}
function appendMessageSync(msg: any) {
if (loggingPaused) {
return;
}
appendFileSync(buildLogFileNameWithCounter(), JSON.stringify(msg) + '\n');
}
async function appendMessage(msg: any) {
if (loggingPaused) {
return;
}
await appendFile(buildLogFileNameWithCounter(), JSON.stringify(msg) + '\n');
}
const messageEventBusLogWriterWorker = {
async appendMessageToLog(msg: any) {
if (syncFileAccess) {
appendMessageSync(msg);
} else {
await appendMessage(msg);
}
},
async confirmMessageSent(confirm: unknown) {
if (syncFileAccess) {
appendMessageSync(confirm);
} else {
await appendMessage(confirm);
}
},
pauseLogging() {
loggingPaused = true;
clearInterval(fileStatTimer);
},
initialize(
basePath: string,
useSyncFileAccess = false,
keepNumberOfFiles = 10,
maxSizeInKB = 102400,
) {
setLogFileBasePath(basePath);
setUseSyncFileAccess(useSyncFileAccess);
setKeepFiles(keepNumberOfFiles);
setMaxLogFileSizeInKB(maxSizeInKB);
},
startLogging() {
if (logFileBasePath) {
renameAndCreateLogs();
loggingPaused = false;
fileStatTimer = setInterval(async () => {
await checkFileSize(buildLogFileNameWithCounter());
}, 5000);
}
},
getLogFileName(counter?: number) {
if (logFileBasePath) {
return buildLogFileNameWithCounter(counter);
} else {
return undefined;
}
},
cleanLogs() {
cleanAllLogs();
},
};
if (isWorkerRuntime()) {
// Register the serializer on the worker thread
expose(messageEventBusLogWriterWorker);
}
export type MessageEventBusLogWriterWorker = typeof messageEventBusLogWriterWorker;

View File

@@ -0,0 +1,219 @@
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-explicit-any */
import express from 'express';
import { ResponseHelper } from '..';
import { isEventMessageOptions } from './EventMessageClasses/AbstractEventMessage';
import { EventMessageGeneric } from './EventMessageClasses/EventMessageGeneric';
import {
EventMessageWorkflow,
EventMessageWorkflowOptions,
} from './EventMessageClasses/EventMessageWorkflow';
import { eventBus, EventMessageReturnMode } from './MessageEventBus/MessageEventBus';
import {
isMessageEventBusDestinationSentryOptions,
MessageEventBusDestinationSentry,
} from './MessageEventBusDestination/MessageEventBusDestinationSentry.ee';
import {
isMessageEventBusDestinationSyslogOptions,
MessageEventBusDestinationSyslog,
} from './MessageEventBusDestination/MessageEventBusDestinationSyslog.ee';
import { MessageEventBusDestinationWebhook } from './MessageEventBusDestination/MessageEventBusDestinationWebhook.ee';
import { eventNamesAll } from './EventMessageClasses';
import {
EventMessageAudit,
EventMessageAuditOptions,
} from './EventMessageClasses/EventMessageAudit';
import { BadRequestError } from '../ResponseHelper';
import {
MessageEventBusDestinationTypeNames,
MessageEventBusDestinationWebhookOptions,
EventMessageTypeNames,
MessageEventBusDestinationOptions,
} from 'n8n-workflow';
import { User } from '../databases/entities/User';
export const eventBusRouter = express.Router();
// ----------------------------------------
// TypeGuards
// ----------------------------------------
const isWithIdString = (candidate: unknown): candidate is { id: string } => {
const o = candidate as { id: string };
if (!o) return false;
return o.id !== undefined;
};
const isWithQueryString = (candidate: unknown): candidate is { query: string } => {
const o = candidate as { query: string };
if (!o) return false;
return o.query !== undefined;
};
// TODO: add credentials
const isMessageEventBusDestinationWebhookOptions = (
candidate: unknown,
): candidate is MessageEventBusDestinationWebhookOptions => {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const o = candidate as MessageEventBusDestinationWebhookOptions;
if (!o) return false;
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
return o.url !== undefined;
};
const isMessageEventBusDestinationOptions = (
candidate: unknown,
): candidate is MessageEventBusDestinationOptions => {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const o = candidate as MessageEventBusDestinationOptions;
if (!o) return false;
return o.__type !== undefined;
};
// ----------------------------------------
// Events
// ----------------------------------------
eventBusRouter.get(
'/event',
ResponseHelper.send(async (req: express.Request): Promise<any> => {
if (isWithQueryString(req.query)) {
switch (req.query.query as EventMessageReturnMode) {
case 'sent':
return eventBus.getEventsSent();
case 'unsent':
return eventBus.getEventsUnsent();
case 'all':
default:
}
}
return eventBus.getEvents();
}),
);
eventBusRouter.post(
'/event',
ResponseHelper.send(async (req: express.Request): Promise<any> => {
if (isEventMessageOptions(req.body)) {
let msg;
switch (req.body.__type) {
case EventMessageTypeNames.workflow:
msg = new EventMessageWorkflow(req.body as EventMessageWorkflowOptions);
break;
case EventMessageTypeNames.audit:
msg = new EventMessageAudit(req.body as EventMessageAuditOptions);
break;
case EventMessageTypeNames.generic:
default:
msg = new EventMessageGeneric(req.body);
}
await eventBus.send(msg);
} else {
throw new BadRequestError(
'Body is not a serialized EventMessage or eventName does not match format {namespace}.{domain}.{event}',
);
}
}),
);
// ----------------------------------------
// Destinations
// ----------------------------------------
eventBusRouter.get(
'/destination',
// eslint-disable-next-line @typescript-eslint/no-unused-vars
ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<any> => {
let result = [];
if (isWithIdString(req.query)) {
result = await eventBus.findDestination(req.query.id);
} else {
result = await eventBus.findDestination();
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return result;
}),
);
eventBusRouter.post(
'/destination',
// eslint-disable-next-line @typescript-eslint/no-unused-vars
ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<any> => {
if (!req.user || (req.user as User).globalRole.name !== 'owner') {
throw new ResponseHelper.UnauthorizedError('Invalid request');
}
if (isMessageEventBusDestinationOptions(req.body)) {
let result;
switch (req.body.__type) {
case MessageEventBusDestinationTypeNames.sentry:
if (isMessageEventBusDestinationSentryOptions(req.body)) {
result = await eventBus.addDestination(new MessageEventBusDestinationSentry(req.body));
}
break;
case MessageEventBusDestinationTypeNames.webhook:
if (isMessageEventBusDestinationWebhookOptions(req.body)) {
result = await eventBus.addDestination(new MessageEventBusDestinationWebhook(req.body));
}
break;
case MessageEventBusDestinationTypeNames.syslog:
if (isMessageEventBusDestinationSyslogOptions(req.body)) {
result = await eventBus.addDestination(new MessageEventBusDestinationSyslog(req.body));
}
break;
default:
throw new BadRequestError(
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
`Body is missing ${req.body.__type} options or type ${req.body.__type} is unknown`,
);
}
if (result) {
await result.saveToDb();
return result;
}
throw new BadRequestError('There was an error adding the destination');
}
throw new BadRequestError('Body is not configuring MessageEventBusDestinationOptions');
}),
);
eventBusRouter.get(
'/testmessage',
// eslint-disable-next-line @typescript-eslint/no-unused-vars
ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<any> => {
let result = false;
if (isWithIdString(req.query)) {
result = await eventBus.testDestination(req.query.id);
}
return result;
}),
);
eventBusRouter.delete(
'/destination',
// eslint-disable-next-line @typescript-eslint/no-unused-vars
ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<any> => {
if (!req.user || (req.user as User).globalRole.name !== 'owner') {
throw new ResponseHelper.UnauthorizedError('Invalid request');
}
if (isWithIdString(req.query)) {
const result = await eventBus.removeDestination(req.query.id);
if (result) {
return result;
}
} else {
throw new BadRequestError('Query is missing id');
}
}),
);
// ----------------------------------------
// Utilities
// ----------------------------------------
eventBusRouter.get(
'/eventnames',
ResponseHelper.send(async (): Promise<any> => {
return eventNamesAll;
}),
);

View File

@@ -0,0 +1 @@
export { eventBus } from './MessageEventBus/MessageEventBus';