✨ Introduce telemetry (#2099)

* introduce analytics

* add user survey backend

* add user survey backend

* set answers on survey submit

Co-authored-by: Mutasem Aldmour <4711238+mutdmour@users.noreply.github.com>

* change name to personalization

* lint

Co-authored-by: Mutasem Aldmour <4711238+mutdmour@users.noreply.github.com>

* N8n 2495 add personalization modal (#2280)

* update modals

* add onboarding modal

* implement questions

* introduce analytics

* simplify impl

* implement survey handling

* add personalized cateogry

* update modal behavior

* add thank you view

* handle empty cases

* rename modal

* standarize modal names

* update image, add tags to headings

* remove unused file

* remove unused interfaces

* clean up footer spacing

* introduce analytics

* refactor to fix bug

* update endpoint

* set min height

* update stories

* update naming from questions to survey

* remove spacing after core categories

* fix bug in logic

* sort nodes

* rename types

* merge with be

* rename userSurvey

* clean up rest api

* use constants for keys

* use survey keys

* clean up types

* move personalization to its own file

Co-authored-by: ahsan-virani <ahsan.virani@gmail.com>

* Survey new options (#2300)

* split up options

* fix quotes

* remove unused import

* add user created workflow event (#2301)

* simplify env vars

* fix versionCli on FE

* update personalization env

* fix event User opened Credentials panel

* fix select modal spacing

* fix nodes panel event

* fix workflow id in workflow execute event

* improve telemetry error logging

* fix config and stop process events

* add flush call on n8n stop

* ready for release

* improve telemetry process exit

* fix merge

* improve n8n stop events

Co-authored-by: Mutasem Aldmour <4711238+mutdmour@users.noreply.github.com>
Co-authored-by: Mutasem <mutdmour@gmail.com>
Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>

packages/cli/src/Server.ts

@@ -28,17 +28,18 @@ import * as express from 'express';
 import { readFileSync } from 'fs';
 import { dirname as pathDirname, join as pathJoin, resolve as pathResolve } from 'path';
 import {
-	getConnectionManager,
-	In,
-	Like,
 	FindManyOptions,
 	FindOneOptions,
+	getConnectionManager,
+	In,
 	IsNull,
 	LessThanOrEqual,
+	Like,
 	Not,
 } from 'typeorm';
 import * as bodyParser from 'body-parser';
 import * as history from 'connect-history-api-fallback';
+import * as os from 'os';
 // eslint-disable-next-line import/no-extraneous-dependencies
 import * as _ from 'lodash';
 import * as clientOAuth2 from 'client-oauth2';
@@ -74,6 +75,8 @@ import {
 	INodeTypeNameVersion,
 	IRunData,
 	INodeVersionedType,
+	ITelemetryClientConfig,
+	ITelemetrySettings,
 	IWorkflowBase,
 	IWorkflowCredentials,
 	LoggerProxy,
@@ -124,11 +127,13 @@ import {
 	IExecutionsStopData,
 	IExecutionsSummary,
 	IExternalHooksClass,
+	IDiagnosticInfo,
 	IN8nUISettings,
 	IPackageVersions,
 	ITagWithCountDb,
 	IWorkflowExecutionDataProcess,
 	IWorkflowResponse,
+	IPersonalizationSurveyAnswers,
 	LoadNodesAndCredentials,
 	NodeTypes,
 	Push,
@@ -142,9 +147,13 @@ import {
 	WorkflowHelpers,
 	WorkflowRunner,
 } from '.';
+
 import * as config from '../config';
 
 import * as TagHelpers from './TagHelpers';
+import * as PersonalizationSurvey from './PersonalizationSurvey';
+
+import { InternalHooksManager } from './InternalHooksManager';
 import { TagEntity } from './databases/entities/TagEntity';
 import { WorkflowEntity } from './databases/entities/WorkflowEntity';
 import { NameRequest } from './WorkflowHelpers';
@@ -243,6 +252,22 @@ class App {
 
 		const urlBaseWebhook = WebhookHelpers.getWebhookBaseUrl();
 
+		const telemetrySettings: ITelemetrySettings = {
+			enabled: config.get('diagnostics.enabled') as boolean,
+		};
+
+		if (telemetrySettings.enabled) {
+			const conf = config.get('diagnostics.config.frontend') as string;
+			const [key, url] = conf.split(';');
+
+			if (!key || !url) {
+				LoggerProxy.warn('Diagnostics frontend config is invalid');
+				telemetrySettings.enabled = false;
+			}
+
+			telemetrySettings.config = { key, url };
+		}
+
 		this.frontendSettings = {
 			endpointWebhook: this.endpointWebhook,
 			endpointWebhookTest: this.endpointWebhookTest,
@@ -264,6 +289,10 @@ class App {
 				infoUrl: config.get('versionNotifications.infoUrl'),
 			},
 			instanceId: '',
+			telemetry: telemetrySettings,
+			personalizationSurvey: {
+				shouldShow: false,
+			},
 		};
 	}
 
@@ -290,7 +319,13 @@ class App {
 
 		this.versions = await GenericHelpers.getVersions();
 		this.frontendSettings.versionCli = this.versions.cli;
-		this.frontendSettings.instanceId = (await generateInstanceId()) as string;
+
+		this.frontendSettings.instanceId = await UserSettings.getInstanceId();
+
+		this.frontendSettings.personalizationSurvey =
+			await PersonalizationSurvey.preparePersonalizationSurvey();
+
+		InternalHooksManager.init(this.frontendSettings.instanceId);
 
 		await this.externalHooks.run('frontend.settings', [this.frontendSettings]);
 
@@ -458,10 +493,13 @@ class App {
 				};
 
 				jwt.verify(token, getKey, jwtVerifyOptions, (err: jwt.VerifyErrors, decoded: object) => {
-					if (err) ResponseHelper.jwtAuthAuthorizationError(res, 'Invalid token');
-					else if (!isTenantAllowed(decoded))
+					if (err) {
+						ResponseHelper.jwtAuthAuthorizationError(res, 'Invalid token');
+					} else if (!isTenantAllowed(decoded)) {
 						ResponseHelper.jwtAuthAuthorizationError(res, 'Tenant not allowed');
-					else next();
+					} else {
+						next();
+					}
 				});
 			});
 		}
@@ -656,6 +694,7 @@ class App {
 
 					// @ts-ignore
 					savedWorkflow.id = savedWorkflow.id.toString();
+					void InternalHooksManager.getInstance().onWorkflowCreated(newWorkflow as IWorkflowBase);
 					return savedWorkflow;
 				},
 			),
@@ -858,12 +897,12 @@ class App {
 					}
 
 					await this.externalHooks.run('workflow.afterUpdate', [workflow]);
+					void InternalHooksManager.getInstance().onWorkflowSaved(workflow as IWorkflowBase);
 
 					if (workflow.active) {
 						// When the workflow is supposed to be active add it again
 						try {
 							await this.externalHooks.run('workflow.activate', [workflow]);
-
 							await this.activeWorkflowRunner.add(id, isActive ? 'update' : 'activate');
 						} catch (error) {
 							// If workflow could not be activated set it again to inactive
@@ -901,6 +940,7 @@ class App {
 				}
 
 				await Db.collections.Workflow!.delete(id);
+				void InternalHooksManager.getInstance().onWorkflowDeleted(id);
 				await this.externalHooks.run('workflow.afterDelete', [id]);
 
 				return true;
@@ -2601,6 +2641,31 @@ class App {
 			),
 		);
 
+		// ----------------------------------------
+		// User Survey
+		// ----------------------------------------
+
+		// Process personalization survey responses
+		this.app.post(
+			`/${this.restEndpoint}/user-survey`,
+			async (req: express.Request, res: express.Response) => {
+				if (!this.frontendSettings.personalizationSurvey.shouldShow) {
+					ResponseHelper.sendErrorResponse(
+						res,
+						new ResponseHelper.ResponseError('User survey already submitted', undefined, 400),
+						false,
+					);
+				}
+
+				const answers = req.body as IPersonalizationSurveyAnswers;
+				await PersonalizationSurvey.writeSurveyToDisk(answers);
+				this.frontendSettings.personalizationSurvey.shouldShow = false;
+				this.frontendSettings.personalizationSurvey.answers = answers;
+				ResponseHelper.sendSuccessResponse(res, undefined, true, 200);
+				void InternalHooksManager.getInstance().onPersonalizationSurveySubmitted(answers);
+			},
+		);
+
 		// ----------------------------------------
 		// Webhooks
 		// ----------------------------------------
@@ -2810,6 +2875,43 @@ export async function start(): Promise<void> {
 		console.log(`Version: ${versions.cli}`);
 
 		await app.externalHooks.run('n8n.ready', [app]);
+		const cpus = os.cpus();
+		const diagnosticInfo: IDiagnosticInfo = {
+			basicAuthActive: config.get('security.basicAuth.active') as boolean,
+			databaseType: (await GenericHelpers.getConfigValue('database.type')) as DatabaseType,
+			disableProductionWebhooksOnMainProcess:
+				config.get('endpoints.disableProductionWebhooksOnMainProcess') === true,
+			notificationsEnabled: config.get('versionNotifications.enabled') === true,
+			versionCli: versions.cli,
+			systemInfo: {
+				os: {
+					type: os.type(),
+					version: os.version(),
+				},
+				memory: os.totalmem() / 1024,
+				cpus: {
+					count: cpus.length,
+					model: cpus[0].model,
+					speed: cpus[0].speed,
+				},
+			},
+			executionVariables: {
+				executions_process: config.get('executions.process'),
+				executions_mode: config.get('executions.mode'),
+				executions_timeout: config.get('executions.timeout'),
+				executions_timeout_max: config.get('executions.maxTimeout'),
+				executions_data_save_on_error: config.get('executions.saveDataOnError'),
+				executions_data_save_on_success: config.get('executions.saveDataOnSuccess'),
+				executions_data_save_on_progress: config.get('executions.saveExecutionProgress'),
+				executions_data_save_manual_executions: config.get('executions.saveDataManualExecutions'),
+				executions_data_prune: config.get('executions.pruneData'),
+				executions_data_max_age: config.get('executions.pruneDataMaxAge'),
+				executions_data_prune_timeout: config.get('executions.pruneDataTimeout'),
+			},
+			deploymentType: config.get('deployment.type'),
+		};
+
+		void InternalHooksManager.getInstance().onServerStarted(diagnosticInfo);
 	});
 }
 
@@ -2848,14 +2950,3 @@ async function getExecutionsCount(
 	const count = await Db.collections.Execution!.count(countFilter);
 	return { count, estimate: false };
 }
-
-async function generateInstanceId() {
-	const encryptionKey = await UserSettings.getEncryptionKey();
-	const hash = encryptionKey
-		? createHash('sha256')
-				.update(encryptionKey.slice(Math.round(encryptionKey.length / 2)))
-				.digest('hex')
-		: undefined;
-
-	return hash;
-}