feat: Add global event bus (#4860)

* fix branch

* fix deserialize, add filewriter

* add catchAll eventGroup/Name

* adding simple Redis sender and receiver to eventbus

* remove native node threads

* improve eventbus

* refactor and simplify

* more refactoring and syslog client

* more refactor, improved endpoints and eventbus

* remove local broker and receivers from mvp

* destination de/serialization

* create MessageEventBusDestinationEntity

* db migrations, load destinations at startup

* add delete destination endpoint

* pnpm merge and circular import fix

* delete destination fix

* trigger log file shuffle after size reached

* add environment variables for eventbus

* reworking event messages

* serialize to thread fix

* some refactor and lint fixing

* add emit to eventbus

* cleanup and fix sending unsent

* quicksave frontend trial

* initial EventTree vue component

* basic log streaming settings in vue

* http request code merge

* create destination settings modals

* fix eventmessage options types

* credentials are loaded

* fix and clean up frontend code

* move request code to axios

* update lock file

* merge fix

* fix redis build

* move destination interfaces into workflow pkg

* revive sentry as destination

* migration fixes and frontend cleanup

* N8N-5777 / N8N-5789 N8N-5788

* N8N-5784

* N8N-5782 removed event levels

* N8N-5790 sentry destination cleanup

* N8N-5786 and refactoring

* N8N-5809 and refactor/cleanup

* UI fixes and anonymize renaming

* N8N-5837

* N8N-5834

* fix no-items UI issues

* remove card / settings label in modal

* N8N-5842 fix

* disable webhook auth for now and update ui

* change sidebar to tabs

* remove payload option

* extend audit events with more user data

* N8N-5853 and UI revert to sidebar

* remove redis destination

* N8N-5864 / N8N-5868 / N8N-5867 / N8N-5865

* ui and licensing fixes

* add node events and info bubbles to frontend

* ui wording changes

* frontend tests

* N8N-5896 and ee rename

* improves backend tests

* merge fix

* fix backend test

* make linter happy

* remove unnecessary cfg / limit  actions to owners

* fix multiple sentry DSN and anon bug

* eslint fix

* more tests and fixes

* merge fix

* fix workflow audit events

* remove 'n8n.workflow.execution.error' event

* merge fix

* lint fix

* lint fix

* review fixes

* fix merge

* prettier fixes

* merge

* review changes

* use loggerproxy

* remove catch from internal hook promises

* fix tests

* lint fix

* include review PR changes

* review changes

* delete duplicate lines from a bad merge

* decouple log-streaming UI options from public API

* logstreaming -> log-streaming for consistency

* do not make unnecessary api calls when log streaming is disabled

* prevent sentryClient.close() from being called if init failed

* fix the e2e test for log-streaming

* review changes

* cleanup

* use `private` for one last private property

* do not use node prefix package names.. just yet

* remove unused import

* fix the tests

because there is a folder called `events`, tsc-alias is messing up all imports for native events module.
https://github.com/justkey007/tsc-alias/issues/152

Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ <aditya@netroy.in>
This commit is contained in:
Michael Auerswald
2023-01-04 09:47:48 +01:00
committed by GitHub
parent 0795cdb74c
commit b67f803cbe
104 changed files with 5867 additions and 219 deletions

View File

@@ -1,3 +1,6 @@
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
import { snakeCase } from 'change-case';
import { BinaryDataManager } from 'n8n-core';
import {
@@ -15,9 +18,28 @@ import {
ITelemetryUserDeletionData,
IWorkflowDb,
IExecutionTrackProperties,
IWorkflowExecutionDataProcess,
} from '@/Interfaces';
import { Telemetry } from '@/telemetry';
import { RoleService } from './role/role.service';
import { eventBus } from './eventbus';
import { User } from './databases/entities/User';
function userToPayload(user: User): {
userId: string;
_email: string;
_firstName: string;
_lastName: string;
globalRole?: string;
} {
return {
userId: user.id,
_email: user.email,
_firstName: user.firstName,
_lastName: user.lastName,
globalRole: user.globalRole?.name,
};
}
export class InternalHooksClass implements IInternalHooksClass {
private versionCli: string;
@@ -82,29 +104,44 @@ export class InternalHooksClass implements IInternalHooksClass {
);
}
async onWorkflowCreated(
userId: string,
workflow: IWorkflowBase,
publicApi: boolean,
): Promise<void> {
async onWorkflowCreated(user: User, workflow: IWorkflowBase, publicApi: boolean): Promise<void> {
const { nodeGraph } = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes);
return this.telemetry.track('User created workflow', {
user_id: userId,
workflow_id: workflow.id,
node_graph_string: JSON.stringify(nodeGraph),
public_api: publicApi,
});
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.workflow.created',
payload: {
...userToPayload(user),
workflowId: workflow.id,
workflowName: workflow.name,
},
}),
this.telemetry.track('User created workflow', {
user_id: user.id,
workflow_id: workflow.id,
node_graph_string: JSON.stringify(nodeGraph),
public_api: publicApi,
}),
]);
}
async onWorkflowDeleted(userId: string, workflowId: string, publicApi: boolean): Promise<void> {
return this.telemetry.track('User deleted workflow', {
user_id: userId,
workflow_id: workflowId,
public_api: publicApi,
});
async onWorkflowDeleted(user: User, workflowId: string, publicApi: boolean): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.workflow.deleted',
payload: {
...userToPayload(user),
workflowId,
},
}),
this.telemetry.track('User deleted workflow', {
user_id: user.id,
workflow_id: workflowId,
public_api: publicApi,
}),
]);
}
async onWorkflowSaved(userId: string, workflow: IWorkflowDb, publicApi: boolean): Promise<void> {
async onWorkflowSaved(user: User, workflow: IWorkflowDb, publicApi: boolean): Promise<void> {
const { nodeGraph } = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes);
const notesCount = Object.keys(nodeGraph.notes).length;
@@ -113,28 +150,88 @@ export class InternalHooksClass implements IInternalHooksClass {
).length;
let userRole: 'owner' | 'sharee' | undefined = undefined;
if (userId && workflow.id) {
const role = await RoleService.getUserRoleForWorkflow(userId, workflow.id);
if (user.id && workflow.id) {
const role = await RoleService.getUserRoleForWorkflow(user.id, workflow.id);
if (role) {
userRole = role.name === 'owner' ? 'owner' : 'sharee';
}
}
return this.telemetry.track(
'User saved workflow',
{
user_id: userId,
workflow_id: workflow.id,
node_graph_string: JSON.stringify(nodeGraph),
notes_count_overlapping: overlappingCount,
notes_count_non_overlapping: notesCount - overlappingCount,
version_cli: this.versionCli,
num_tags: workflow.tags?.length ?? 0,
public_api: publicApi,
sharing_role: userRole,
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.workflow.updated',
payload: {
...userToPayload(user),
workflowId: workflow.id,
workflowName: workflow.name,
},
}),
this.telemetry.track(
'User saved workflow',
{
user_id: user.id,
workflow_id: workflow.id,
node_graph_string: JSON.stringify(nodeGraph),
notes_count_overlapping: overlappingCount,
notes_count_non_overlapping: notesCount - overlappingCount,
version_cli: this.versionCli,
num_tags: workflow.tags?.length ?? 0,
public_api: publicApi,
sharing_role: userRole,
},
{ withPostHog: true },
),
]);
}
async onNodeBeforeExecute(
executionId: string,
workflow: IWorkflowBase,
nodeName: string,
): Promise<void> {
void eventBus.sendNodeEvent({
eventName: 'n8n.node.started',
payload: {
executionId,
nodeName,
workflowId: workflow.id?.toString(),
workflowName: workflow.name,
},
{ withPostHog: true },
);
});
}
async onNodePostExecute(
executionId: string,
workflow: IWorkflowBase,
nodeName: string,
): Promise<void> {
void eventBus.sendNodeEvent({
eventName: 'n8n.node.finished',
payload: {
executionId,
nodeName,
workflowId: workflow.id?.toString(),
workflowName: workflow.name,
},
});
}
async onWorkflowBeforeExecute(
executionId: string,
data: IWorkflowExecutionDataProcess,
): Promise<void> {
void Promise.all([
eventBus.sendWorkflowEvent({
eventName: 'n8n.workflow.started',
payload: {
executionId,
userId: data.userId,
workflowId: data.workflowData.id?.toString(),
isManual: data.executionMode === 'manual',
workflowName: data.workflowData.name,
},
}),
]);
}
async onWorkflowPostExecute(
@@ -208,6 +305,7 @@ export class InternalHooksClass implements IInternalHooksClass {
let userRole: 'owner' | 'sharee' | undefined = undefined;
if (userId) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
const role = await RoleService.getUserRoleForWorkflow(userId, workflow.id);
if (role) {
userRole = role.name === 'owner' ? 'owner' : 'sharee';
@@ -266,11 +364,39 @@ export class InternalHooksClass implements IInternalHooksClass {
}
}
return Promise.all([
...promises,
BinaryDataManager.getInstance().persistBinaryDataForExecutionId(executionId),
this.telemetry.trackWorkflowExecution(properties),
]).then(() => {});
promises.push(
properties.success
? eventBus.sendWorkflowEvent({
eventName: 'n8n.workflow.success',
payload: {
executionId,
success: properties.success,
userId: properties.user_id,
workflowId: properties.workflow_id,
isManual: properties.is_manual,
workflowName: workflow.name,
},
})
: eventBus.sendWorkflowEvent({
eventName: 'n8n.workflow.failed',
payload: {
executionId,
success: properties.success,
userId: properties.user_id,
workflowId: properties.workflow_id,
lastNodeExecuted: runData?.data.resultData.lastNodeExecuted,
errorNodeType: properties.error_node_type,
errorNodeId: properties.error_node_id?.toString(),
errorMessage: properties.error_message?.toString(),
isManual: properties.is_manual,
workflowName: workflow.name,
},
}),
);
await BinaryDataManager.getInstance().persistBinaryDataForExecutionId(executionId);
void Promise.all([...promises, this.telemetry.trackWorkflowExecution(properties)]);
}
async onWorkflowSharingUpdate(workflowId: string, userId: string, userList: string[]) {
@@ -293,32 +419,66 @@ export class InternalHooksClass implements IInternalHooksClass {
return Promise.race([timeoutPromise, this.telemetry.trackN8nStop()]);
}
async onUserDeletion(
userId: string,
userDeletionData: ITelemetryUserDeletionData,
publicApi: boolean,
): Promise<void> {
return this.telemetry.track('User deleted user', {
...userDeletionData,
user_id: userId,
public_api: publicApi,
});
async onUserDeletion(userDeletionData: {
user: User;
telemetryData: ITelemetryUserDeletionData;
publicApi: boolean;
}): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.deleted',
payload: {
...userToPayload(userDeletionData.user),
},
}),
this.telemetry.track('User deleted user', {
...userDeletionData.telemetryData,
user_id: userDeletionData.user.id,
public_api: userDeletionData.publicApi,
}),
]);
}
async onUserInvite(userInviteData: {
user_id: string;
user: User;
target_user_id: string[];
public_api: boolean;
}): Promise<void> {
return this.telemetry.track('User invited new user', userInviteData);
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.invited',
payload: {
...userToPayload(userInviteData.user),
targetUserId: userInviteData.target_user_id,
},
}),
this.telemetry.track('User invited new user', {
user_id: userInviteData.user.id,
target_user_id: userInviteData.target_user_id,
public_api: userInviteData.public_api,
}),
]);
}
async onUserReinvite(userReinviteData: {
user_id: string;
user: User;
target_user_id: string;
public_api: boolean;
}): Promise<void> {
return this.telemetry.track('User resent new user invite email', userReinviteData);
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.reinvited',
payload: {
...userToPayload(userReinviteData.user),
targetUserId: userReinviteData.target_user_id,
},
}),
this.telemetry.track('User resent new user invite email', {
user_id: userReinviteData.user.id,
target_user_id: userReinviteData.target_user_id,
public_api: userReinviteData.public_api,
}),
]);
}
async onUserRetrievedUser(userRetrievedData: {
@@ -363,19 +523,56 @@ export class InternalHooksClass implements IInternalHooksClass {
return this.telemetry.track('User retrieved all workflows', userRetrievedData);
}
async onUserUpdate(userUpdateData: { user_id: string; fields_changed: string[] }): Promise<void> {
return this.telemetry.track('User changed personal settings', userUpdateData);
async onUserUpdate(userUpdateData: { user: User; fields_changed: string[] }): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.updated',
payload: {
...userToPayload(userUpdateData.user),
fieldsChanged: userUpdateData.fields_changed,
},
}),
this.telemetry.track('User changed personal settings', {
user_id: userUpdateData.user.id,
fields_changed: userUpdateData.fields_changed,
}),
]);
}
async onUserInviteEmailClick(userInviteClickData: { user_id: string }): Promise<void> {
return this.telemetry.track('User clicked invite link from email', userInviteClickData);
async onUserInviteEmailClick(userInviteClickData: {
inviter: User;
invitee: User;
}): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.invitation.accepted',
payload: {
invitee: {
...userToPayload(userInviteClickData.invitee),
},
inviter: {
...userToPayload(userInviteClickData.inviter),
},
},
}),
this.telemetry.track('User clicked invite link from email', {
user_id: userInviteClickData.invitee.id,
}),
]);
}
async onUserPasswordResetEmailClick(userPasswordResetData: { user_id: string }): Promise<void> {
return this.telemetry.track(
'User clicked password reset link from email',
userPasswordResetData,
);
async onUserPasswordResetEmailClick(userPasswordResetData: { user: User }): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.reset',
payload: {
...userToPayload(userPasswordResetData.user),
},
}),
this.telemetry.track('User clicked password reset link from email', {
user_id: userPasswordResetData.user.id,
}),
]);
}
async onUserTransactionalEmail(userTransactionalEmailData: {
@@ -398,44 +595,85 @@ export class InternalHooksClass implements IInternalHooksClass {
return this.telemetry.track('User invoked API', userInvokedApiData);
}
async onApiKeyDeleted(apiKeyDeletedData: {
user_id: string;
public_api: boolean;
}): Promise<void> {
return this.telemetry.track('API key deleted', apiKeyDeletedData);
async onApiKeyDeleted(apiKeyDeletedData: { user: User; public_api: boolean }): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.api.deleted',
payload: {
...userToPayload(apiKeyDeletedData.user),
},
}),
this.telemetry.track('API key deleted', {
user_id: apiKeyDeletedData.user.id,
public_api: apiKeyDeletedData.public_api,
}),
]);
}
async onApiKeyCreated(apiKeyCreatedData: {
user_id: string;
public_api: boolean;
}): Promise<void> {
return this.telemetry.track('API key created', apiKeyCreatedData);
async onApiKeyCreated(apiKeyCreatedData: { user: User; public_api: boolean }): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.api.created',
payload: {
...userToPayload(apiKeyCreatedData.user),
},
}),
this.telemetry.track('API key created', {
user_id: apiKeyCreatedData.user.id,
public_api: apiKeyCreatedData.public_api,
}),
]);
}
async onUserPasswordResetRequestClick(userPasswordResetData: { user_id: string }): Promise<void> {
return this.telemetry.track(
'User requested password reset while logged out',
userPasswordResetData,
);
async onUserPasswordResetRequestClick(userPasswordResetData: { user: User }): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.reset.requested',
payload: {
...userToPayload(userPasswordResetData.user),
},
}),
this.telemetry.track('User requested password reset while logged out', {
user_id: userPasswordResetData.user.id,
}),
]);
}
async onInstanceOwnerSetup(instanceOwnerSetupData: { user_id: string }): Promise<void> {
return this.telemetry.track('Owner finished instance setup', instanceOwnerSetupData);
}
async onUserSignup(userSignupData: { user_id: string }): Promise<void> {
return this.telemetry.track('User signed up', userSignupData);
async onUserSignup(userSignupData: { user: User }): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.signedup',
payload: {
...userToPayload(userSignupData.user),
},
}),
this.telemetry.track('User signed up', {
user_id: userSignupData.user.id,
}),
]);
}
async onEmailFailed(failedEmailData: {
user_id: string;
user: User;
message_type: 'Reset password' | 'New user invite' | 'Resend invite';
public_api: boolean;
}): Promise<void> {
return this.telemetry.track(
'Instance failed to send transactional email to user',
failedEmailData,
);
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.email.failed',
payload: {
messageType: failedEmailData.message_type,
...userToPayload(failedEmailData.user),
},
}),
this.telemetry.track('Instance failed to send transactional email to user', {
user_id: failedEmailData.user.id,
}),
]);
}
/**
@@ -443,27 +681,63 @@ export class InternalHooksClass implements IInternalHooksClass {
*/
async onUserCreatedCredentials(userCreatedCredentialsData: {
user: User;
credential_name: string;
credential_type: string;
credential_id: string;
public_api: boolean;
}): Promise<void> {
return this.telemetry.track('User created credentials', {
...userCreatedCredentialsData,
instance_id: this.instanceId,
});
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.credentials.created',
payload: {
...userToPayload(userCreatedCredentialsData.user),
credentialName: userCreatedCredentialsData.credential_name,
credentialType: userCreatedCredentialsData.credential_type,
credentialId: userCreatedCredentialsData.credential_id,
},
}),
this.telemetry.track('User created credentials', {
user_id: userCreatedCredentialsData.user.id,
credential_type: userCreatedCredentialsData.credential_type,
credential_id: userCreatedCredentialsData.credential_id,
instance_id: this.instanceId,
}),
]);
}
async onUserSharedCredentials(userSharedCredentialsData: {
user: User;
credential_name: string;
credential_type: string;
credential_id: string;
user_id_sharer: string;
user_ids_sharees_added: string[];
sharees_removed: number | null;
}): Promise<void> {
return this.telemetry.track('User updated cred sharing', {
...userSharedCredentialsData,
instance_id: this.instanceId,
});
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.credentials.shared',
payload: {
...userToPayload(userSharedCredentialsData.user),
credentialName: userSharedCredentialsData.credential_name,
credentialType: userSharedCredentialsData.credential_type,
credentialId: userSharedCredentialsData.credential_id,
userIdSharer: userSharedCredentialsData.user_id_sharer,
userIdsShareesAdded: userSharedCredentialsData.user_ids_sharees_added,
shareesRemoved: userSharedCredentialsData.sharees_removed,
},
}),
this.telemetry.track('User updated cred sharing', {
user_id: userSharedCredentialsData.user.id,
credential_type: userSharedCredentialsData.credential_type,
credential_id: userSharedCredentialsData.credential_id,
user_id_sharer: userSharedCredentialsData.user_id_sharer,
user_ids_sharees_added: userSharedCredentialsData.user_ids_sharees_added,
sharees_removed: userSharedCredentialsData.sharees_removed,
instance_id: this.instanceId,
}),
]);
}
/**
@@ -471,7 +745,7 @@ export class InternalHooksClass implements IInternalHooksClass {
*/
async onCommunityPackageInstallFinished(installationData: {
user_id: string;
user: User;
input_string: string;
package_name: string;
success: boolean;
@@ -481,11 +755,37 @@ export class InternalHooksClass implements IInternalHooksClass {
package_author_email?: string;
failure_reason?: string;
}): Promise<void> {
return this.telemetry.track('cnr package install finished', installationData);
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.package.installed',
payload: {
...userToPayload(installationData.user),
inputString: installationData.input_string,
packageName: installationData.package_name,
success: installationData.success,
packageVersion: installationData.package_version,
packageNodeNames: installationData.package_node_names,
packageAuthor: installationData.package_author,
packageAuthorEmail: installationData.package_author_email,
failureReason: installationData.failure_reason,
},
}),
this.telemetry.track('cnr package install finished', {
user_id: installationData.user.id,
input_string: installationData.input_string,
package_name: installationData.package_name,
success: installationData.success,
package_version: installationData.package_version,
package_node_names: installationData.package_node_names,
package_author: installationData.package_author,
package_author_email: installationData.package_author_email,
failure_reason: installationData.failure_reason,
}),
]);
}
async onCommunityPackageUpdateFinished(updateData: {
user_id: string;
user: User;
package_name: string;
package_version_current: string;
package_version_new: string;
@@ -493,18 +793,60 @@ export class InternalHooksClass implements IInternalHooksClass {
package_author?: string;
package_author_email?: string;
}): Promise<void> {
return this.telemetry.track('cnr package updated', updateData);
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.package.updated',
payload: {
...userToPayload(updateData.user),
packageName: updateData.package_name,
packageVersionCurrent: updateData.package_version_current,
packageVersionNew: updateData.package_version_new,
packageNodeNames: updateData.package_node_names,
packageAuthor: updateData.package_author,
packageAuthorEmail: updateData.package_author_email,
},
}),
this.telemetry.track('cnr package updated', {
user_id: updateData.user.id,
package_name: updateData.package_name,
package_version_current: updateData.package_version_current,
package_version_new: updateData.package_version_new,
package_node_names: updateData.package_node_names,
package_author: updateData.package_author,
package_author_email: updateData.package_author_email,
}),
]);
}
async onCommunityPackageDeleteFinished(updateData: {
user_id: string;
async onCommunityPackageDeleteFinished(deleteData: {
user: User;
package_name: string;
package_version: string;
package_node_names: string[];
package_author?: string;
package_author_email?: string;
}): Promise<void> {
return this.telemetry.track('cnr package deleted', updateData);
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.package.deleted',
payload: {
...userToPayload(deleteData.user),
packageName: deleteData.package_name,
packageVersion: deleteData.package_version,
packageNodeNames: deleteData.package_node_names,
packageAuthor: deleteData.package_author,
packageAuthorEmail: deleteData.package_author_email,
},
}),
this.telemetry.track('cnr package deleted', {
user_id: deleteData.user.id,
package_name: deleteData.package_name,
package_version: deleteData.package_version,
package_node_names: deleteData.package_node_names,
package_author: deleteData.package_author,
package_author_email: deleteData.package_author_email,
}),
]);
}
/**