feat(benchmark): Report benchmark results to a configurable webhook (#10754)

packages/@n8n/benchmark/src/testExecution/k6Executor.ts

@@ -1,12 +1,10 @@
 import fs from 'fs';
 import path from 'path';
+import assert from 'node:assert/strict';
 import { $, which, tmpfile } from 'zx';
 import type { Scenario } from '@/types/scenario';
-
-export type K6Tag = {
-	name: string;
-	value: string;
-};
+import { buildTestReport, type K6Tag } from '@/testExecution/testReport';
+export type { K6Tag };
 
 export type K6ExecutorOpts = {
 	k6ExecutablePath: string;
@@ -17,6 +15,10 @@ export type K6ExecutorOpts = {
 	k6ApiToken?: string;
 	n8nApiBaseUrl: string;
 	tags?: K6Tag[];
+	resultsWebhook?: {
+		url: string;
+		authHeader: string;
+	};
 };
 
 export type K6RunOpts = {
@@ -61,7 +63,7 @@ export function handleSummary(data) {
 			['--vus', this.opts.vus],
 		];
 
-		if (this.opts.k6ApiToken) {
+		if (!this.opts.resultsWebhook && this.opts.k6ApiToken) {
 			flags.push(['--out', 'cloud']);
 		}
 
@@ -69,20 +71,46 @@ export function handleSummary(data) {
 
 		const k6ExecutablePath = await this.resolveK6ExecutablePath();
 
-		const processPromise = $({
+		await $({
 			cwd: runDirPath,
 			env: {
 				API_BASE_URL: this.opts.n8nApiBaseUrl,
 				K6_CLOUD_TOKEN: this.opts.k6ApiToken,
 				SCRIPT_FILE_PATH: augmentedTestScriptPath,
 			},
+			stdio: 'inherit',
 		})`${k6ExecutablePath} run ${flattedFlags} ${augmentedTestScriptPath}`;
 
-		for await (const chunk of processPromise.stdout) {
-			console.log((chunk as Buffer).toString());
-		}
+		console.log('\n');
 
-		this.loadEndOfTestSummary(runDirPath, scenarioRunName);
+		if (this.opts.resultsWebhook) {
+			const endOfTestSummary = this.loadEndOfTestSummary(runDirPath, scenarioRunName);
+
+			const testReport = buildTestReport(scenario, endOfTestSummary, [
+				...(this.opts.tags ?? []),
+				{ name: 'Vus', value: this.opts.vus.toString() },
+				{ name: 'Duration', value: this.opts.duration.toString() },
+			]);
+
+			await this.sendTestReport(testReport);
+		}
+	}
+
+	async sendTestReport(testReport: unknown) {
+		assert(this.opts.resultsWebhook);
+
+		const response = await fetch(this.opts.resultsWebhook.url, {
+			method: 'POST',
+			body: JSON.stringify(testReport),
+			headers: {
+				Authorization: this.opts.resultsWebhook.authHeader,
+				'Content-Type': 'application/json',
+			},
+		});
+
+		if (!response.ok) {
+			console.warn(`Failed to send test summary: ${response.status} ${await response.text()}`);
+		}
 	}
 
 	/**

packages/@n8n/benchmark/src/testExecution/k6Summary.ts

@@ -183,7 +183,7 @@ interface CounterValues {
 	rate: number;
 }
 
-interface TrendMetric {
+interface K6TrendMetric {
 	type: 'trend';
 	contains: 'time';
 	values: TrendValues;
@@ -195,7 +195,7 @@ interface RateMetric {
 	values: RateValues;
 }
 
-interface CounterMetric {
+interface K6CounterMetric {
 	type: 'counter';
 	contains: MetricContains;
 	values: CounterValues;
@@ -214,24 +214,24 @@ interface State {
 }
 
 interface Metrics {
-	http_req_tls_handshaking: TrendMetric;
+	http_req_tls_handshaking: K6TrendMetric;
 	checks: RateMetric;
-	http_req_sending: TrendMetric;
-	http_reqs: CounterMetric;
-	http_req_blocked: TrendMetric;
-	data_received: CounterMetric;
-	iterations: CounterMetric;
-	http_req_waiting: TrendMetric;
-	http_req_receiving: TrendMetric;
-	'http_req_duration{expected_response:true}': TrendMetric;
-	iteration_duration: TrendMetric;
-	http_req_connecting: TrendMetric;
+	http_req_sending: K6TrendMetric;
+	http_reqs: K6CounterMetric;
+	http_req_blocked: K6TrendMetric;
+	data_received: K6CounterMetric;
+	iterations: K6CounterMetric;
+	http_req_waiting: K6TrendMetric;
+	http_req_receiving: K6TrendMetric;
+	'http_req_duration{expected_response:true}': K6TrendMetric;
+	iteration_duration: K6TrendMetric;
+	http_req_connecting: K6TrendMetric;
 	http_req_failed: RateMetric;
-	http_req_duration: TrendMetric;
-	data_sent: CounterMetric;
+	http_req_duration: K6TrendMetric;
+	data_sent: K6CounterMetric;
 }
 
-interface Check {
+interface K6Check {
 	name: string;
 	path: string;
 	id: string;
@@ -244,7 +244,7 @@ interface RootGroup {
 	path: string;
 	id: string;
 	groups: unknown[];
-	checks: Check[];
+	checks: K6Check[];
 }
 
 interface K6EndOfTestSummary {

packages/@n8n/benchmark/src/testExecution/testReport.ts

@@ -0,0 +1,102 @@
+import { nanoid } from 'nanoid';
+import type { Scenario } from '@/types/scenario';
+
+export type K6Tag = {
+	name: string;
+	value: string;
+};
+
+export type Check = {
+	name: string;
+	passes: number;
+	fails: number;
+};
+
+export type CounterMetric = {
+	type: 'counter';
+	count: number;
+	rate: number;
+};
+
+export type TrendMetric = {
+	type: 'trend';
+	'p(95)': number;
+	avg: number;
+	min: number;
+	med: number;
+	max: number;
+	'p(90)': number;
+};
+
+export type TestReport = {
+	runId: string;
+	ts: string; // ISO8601
+	scenarioName: string;
+	tags: K6Tag[];
+	metrics: {
+		iterations: CounterMetric;
+		dataReceived: CounterMetric;
+		dataSent: CounterMetric;
+		httpRequests: CounterMetric;
+		httpRequestDuration: TrendMetric;
+		httpRequestSending: TrendMetric;
+		httpRequestReceiving: TrendMetric;
+		httpRequestWaiting: TrendMetric;
+	};
+	checks: Check[];
+};
+
+function k6CheckToCheck(check: K6Check): Check {
+	return {
+		name: check.name,
+		passes: check.passes,
+		fails: check.fails,
+	};
+}
+
+function k6CounterToCounter(counter: K6CounterMetric): CounterMetric {
+	return {
+		type: 'counter',
+		count: counter.values.count,
+		rate: counter.values.rate,
+	};
+}
+
+function k6TrendToTrend(trend: K6TrendMetric): TrendMetric {
+	return {
+		type: 'trend',
+		'p(90)': trend.values['p(90)'],
+		avg: trend.values.avg,
+		min: trend.values.min,
+		med: trend.values.med,
+		max: trend.values.max,
+		'p(95)': trend.values['p(95)'],
+	};
+}
+
+/**
+ * Converts the k6 test summary to a test report
+ */
+export function buildTestReport(
+	scenario: Scenario,
+	endOfTestSummary: K6EndOfTestSummary,
+	tags: K6Tag[],
+): TestReport {
+	return {
+		runId: nanoid(),
+		ts: new Date().toISOString(),
+		scenarioName: scenario.name,
+		tags,
+		checks: endOfTestSummary.root_group.checks.map(k6CheckToCheck),
+		metrics: {
+			dataReceived: k6CounterToCounter(endOfTestSummary.metrics.data_received),
+			dataSent: k6CounterToCounter(endOfTestSummary.metrics.data_sent),
+			httpRequests: k6CounterToCounter(endOfTestSummary.metrics.http_reqs),
+			httpRequestDuration: k6TrendToTrend(endOfTestSummary.metrics.http_req_duration),
+			httpRequestSending: k6TrendToTrend(endOfTestSummary.metrics.http_req_sending),
+			httpRequestReceiving: k6TrendToTrend(endOfTestSummary.metrics.http_req_receiving),
+			httpRequestWaiting: k6TrendToTrend(endOfTestSummary.metrics.http_req_waiting),
+			iterations: k6CounterToCounter(endOfTestSummary.metrics.iterations),
+		},
+	};
+}