feat(API): Report unhandled app crashes to Sentry (#4548)

* SIGTERM/SIGINT should only be handled once

* move error-handling initialization to commands

* create a new `sleep` function in workflow utils

* detect crashes and report them to Sentry

packages/cli/src/CrashJournal.ts

@@ -0,0 +1,33 @@
+import { existsSync } from 'fs';
+import { mkdir, utimes, open, rm } from 'fs/promises';
+import { join, dirname } from 'path';
+import { UserSettings } from 'n8n-core';
+import { ErrorReporterProxy as ErrorReporter, LoggerProxy, sleep } from 'n8n-workflow';
+
+export const touchFile = async (filePath: string): Promise<void> => {
+	await mkdir(dirname(filePath), { recursive: true });
+	const time = new Date();
+	try {
+		await utimes(filePath, time, time);
+	} catch {
+		const fd = await open(filePath, 'w');
+		await fd.close();
+	}
+};
+
+const journalFile = join(UserSettings.getUserN8nFolderPath(), 'crash.journal');
+
+export const init = async () => {
+	if (existsSync(journalFile)) {
+		// Crash detected
+		ErrorReporter.warn('Last session crashed');
+		LoggerProxy.error('Last session crashed');
+		// add a 10 seconds pause to slow down crash-looping
+		await sleep(10_000);
+	}
+	await touchFile(journalFile);
+};
+
+export const cleanup = async () => {
+	await rm(journalFile, { force: true });
+};

packages/cli/src/ErrorReporting.ts

@@ -6,7 +6,7 @@ import { ErrorReporterProxy } from 'n8n-workflow';
 
 let initialized = false;
 
-export const initErrorHandling = (app?: Application) => {
+export const initErrorHandling = () => {
 	if (initialized) return;
 
 	if (!config.getEnv('diagnostics.enabled')) {
@@ -27,15 +27,15 @@ export const initErrorHandling = (app?: Application) => {
 		},
 	});
 
-	if (app) {
-		const { requestHandler, errorHandler } = Sentry.Handlers;
-		app.use(requestHandler());
-		app.use(errorHandler());
-	}
-
 	ErrorReporterProxy.init({
 		report: (error, options) => Sentry.captureException(error, options),
 	});
 
 	initialized = true;
 };
+
+export const setupErrorMiddleware = (app: Application) => {
+	const { requestHandler, errorHandler } = Sentry.Handlers;
+	app.use(requestHandler());
+	app.use(errorHandler());
+};

packages/cli/src/Server.ts

@@ -155,7 +155,7 @@ import glob from 'fast-glob';
 import { ResponseError } from './ResponseHelper';
 
 import { toHttpNodeParameters } from './CurlConverterHelper';
-import { initErrorHandling } from './ErrorReporting';
+import { setupErrorMiddleware } from './ErrorReporting';
 
 require('body-parser-xml')(bodyParser);
 
@@ -259,7 +259,7 @@ class App {
 		this.presetCredentialsLoaded = false;
 		this.endpointPresetCredentials = config.getEnv('credentials.overwrite.endpoint');
 
-		initErrorHandling(this.app);
+		setupErrorMiddleware(this.app);
 
 		const urlBaseWebhook = WebhookHelpers.getWebhookBaseUrl();
 		const telemetrySettings: ITelemetrySettings = {

packages/cli/src/WebhookServer.ts

@@ -33,7 +33,7 @@ import {
 import config from '../config';
 // eslint-disable-next-line import/no-cycle
 import { WEBHOOK_METHODS } from './WebhookHelpers';
-import { initErrorHandling } from './ErrorReporting';
+import { setupErrorMiddleware } from './ErrorReporting';
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-unsafe-call
 require('body-parser-xml')(bodyParser);
@@ -219,7 +219,7 @@ class App {
 		this.presetCredentialsLoaded = false;
 		this.endpointPresetCredentials = config.getEnv('credentials.overwrite.endpoint');
 
-		initErrorHandling(this.app);
+		setupErrorMiddleware(this.app);
 	}
 
 	/**

packages/cli/src/WorkflowRunnerProcess.ts

@@ -86,8 +86,8 @@ export class WorkflowRunnerProcess {
 	}
 
 	async runWorkflow(inputData: IWorkflowExecutionDataProcessWithExecution): Promise<IRun> {
-		process.on('SIGTERM', WorkflowRunnerProcess.stopProcess);
-		process.on('SIGINT', WorkflowRunnerProcess.stopProcess);
+		process.once('SIGTERM', WorkflowRunnerProcess.stopProcess);
+		process.once('SIGINT', WorkflowRunnerProcess.stopProcess);
 
 		// eslint-disable-next-line no-multi-assign
 		const logger = (this.logger = getLogger());