feat: Add benchmarking of pooled sqlite (no-changelog) (#10550)

packages/@n8n/benchmark/scripts/runInCloud.mjs

@@ -15,7 +15,7 @@
 // @ts-check
 import fs from 'fs';
 import minimist from 'minimist';
-import { $, sleep, which } from 'zx';
+import { sleep, which } from 'zx';
 import path from 'path';
 import { SshClient } from './sshClient.mjs';
 import { TerraformClient } from './terraformClient.mjs';
@@ -61,7 +61,6 @@ async function ensureDependencies() {
 }
 
 /**
- *
  * @param {Config} config
  * @param {BenchmarkEnv} benchmarkEnv
  */
@@ -86,7 +85,32 @@ async function runBenchmarksOnVm(config, benchmarkEnv) {
 	// Give some time for the VM to be ready
 	await sleep(1000);
 
-	console.log('Running benchmarks...');
+	if (config.n8nSetupToUse === 'all') {
+		const availableSetups = readAvailableN8nSetups();
+
+		for (const n8nSetup of availableSetups) {
+			await runBenchmarkForN8nSetup({
+				config,
+				sshClient,
+				scriptsDir,
+				n8nSetup,
+			});
+		}
+	} else {
+		await runBenchmarkForN8nSetup({
+			config,
+			sshClient,
+			scriptsDir,
+			n8nSetup: config.n8nSetupToUse,
+		});
+	}
+}
+
+/**
+ * @param {{ config: Config; sshClient: any; scriptsDir: string; n8nSetup: string; }} opts
+ */
+async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup }) {
+	console.log(`Running benchmarks for ${n8nSetup}...`);
 	const runScriptPath = path.join(scriptsDir, 'runOnVm.mjs');
 
 	const flags = {
@@ -100,7 +124,7 @@ async function runBenchmarksOnVm(config, benchmarkEnv) {
 		.map(([key, value]) => `--${key}=${value}`)
 		.join(' ');
 
-	await sshClient.ssh(`npx zx ${runScriptPath} ${flagsString} ${config.n8nSetupToUse}`, {
+	await sshClient.ssh(`npx zx ${runScriptPath} ${flagsString} ${n8nSetup}`, {
 		// Test run should always log its output
 		verbose: true,
 	});
@@ -138,10 +162,15 @@ function readAvailableN8nSetups() {
  * @returns {Promise<Config>}
  */
 async function parseAndValidateConfig() {
-	const args = minimist(process.argv.slice(2), {
-		boolean: ['debug'],
+	const args = minimist(process.argv.slice(3), {
+		boolean: ['debug', 'help'],
 	});
 
+	if (args.help) {
+		printUsage();
+		process.exit(0);
+	}
+
 	const n8nSetupToUse = await getAndValidateN8nSetup(args);
 	const isVerbose = args.debug || false;
 	const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest';
@@ -163,10 +192,8 @@ async function parseAndValidateConfig() {
 async function getAndValidateN8nSetup(args) {
 	// Last parameter is the n8n setup to use
 	const n8nSetupToUse = args._[args._.length - 1];
-
-	if (!n8nSetupToUse) {
-		printUsage();
-		process.exit(1);
+	if (!n8nSetupToUse || n8nSetupToUse === 'all') {
+		return 'all';
 	}
 
 	const availableSetups = readAvailableN8nSetups();
@@ -182,19 +209,20 @@ async function getAndValidateN8nSetup(args) {
 function printUsage() {
 	const availableSetups = readAvailableN8nSetups();
 
-	console.log('Usage: zx scripts/runInCloud.mjs <n8n setup name>');
-	console.log('   eg: zx scripts/runInCloud.mjs sqlite');
+	console.log('Usage: zx scripts/runInCloud.mjs [n8n setup name]');
+	console.log('   eg: zx scripts/runInCloud.mjs');
 	console.log('');
 	console.log('Options:');
+	console.log(
+		`  [n8n setup name]     Against which n8n setup to run the benchmarks. One of: ${['all', ...availableSetups].join(', ')}. Default is all`,
+	);
 	console.log('  --debug              Enable verbose output');
 	console.log('  --n8nTag             Docker tag for n8n image. Default is latest');
 	console.log('  --benchmarkTag       Docker tag for benchmark cli image. Default is latest');
 	console.log(
-		'  --k6ApiToken         API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used.',
+		'  --k6ApiToken         API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used',
 	);
 	console.log('');
-	console.log('Available setups:');
-	console.log(`  ${availableSetups.join(', ')}`);
 }
 
 main().catch(console.error);

packages/@n8n/benchmark/scripts/runOnVm/bootstrap.sh

@@ -10,8 +10,8 @@ CURRENT_USER=$(whoami)
 # Mount the data disk
 if [ -d "/n8n" ]; then
 	echo "Data disk already mounted. Clearing it..."
-	rm -rf /n8n/*
-	rm -rf /n8n/.[!.]*
+	sudo rm -rf /n8n/*
+	sudo rm -rf /n8n/.[!.]*
 else
 	sudo mkdir -p /n8n
 	sudo parted /dev/sdc --script mklabel gpt mkpart xfspart xfs 0% 100%

packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite-legacy/docker-compose.yml

@@ -0,0 +1,17 @@
+services:
+  n8n:
+    image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest}
+    environment:
+      - N8N_DIAGNOSTICS_ENABLED=false
+      - N8N_USER_FOLDER=/n8n
+    ports:
+      - 5678:5678
+    volumes:
+      - /n8n:/n8n
+  benchmark:
+    image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest}
+    depends_on:
+      - n8n
+    environment:
+      - N8N_BASE_URL=http://n8n:5678
+      - K6_API_TOKEN=${K6_API_TOKEN}

packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite/docker-compose.yml

@@ -4,6 +4,8 @@ services:
     environment:
       - N8N_DIAGNOSTICS_ENABLED=false
       - N8N_USER_FOLDER=/n8n
+      - DB_SQLITE_POOL_SIZE=3
+      - DB_SQLITE_ENABLE_WAL=true
     ports:
       - 5678:5678
     volumes:

packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs

@@ -2,23 +2,19 @@
 /**
  * This script runs the benchmarks using a given docker compose setup
  */
+// @ts-check
+import path from 'path';
+import { $, argv, fs } from 'zx';
 
-import { $ } from 'zx';
-
-const [n8nSetupToUse] = argv._;
-
-if (!n8nSetupToUse) {
-	printUsage();
-	process.exit(1);
-}
-
-function printUsage() {
-	console.log('Usage: zx runOnVm.mjs <envName>');
-	console.log('   eg: zx runOnVm.mjs sqlite');
-}
+const paths = {
+	n8nSetupsDir: path.join(__dirname, 'n8nSetups'),
+};
 
 async function main() {
-	const composeFilePath = path.join(__dirname, 'n8nSetups', n8nSetupToUse);
+	const [n8nSetupToUse] = argv._;
+	validateN8nSetup(n8nSetupToUse);
+
+	const composeFilePath = path.join(paths.n8nSetupsDir, n8nSetupToUse);
 	const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest';
 	const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
 	const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined;
@@ -30,6 +26,7 @@ async function main() {
 			N8N_VERSION: n8nTag,
 			BENCHMARK_VERSION: benchmarkTag,
 			K6_API_TOKEN: k6ApiToken,
+			N8N_BENCHMARK_SCENARIO_NAME_PREFIX: n8nSetupToUse,
 		},
 	});
 
@@ -52,4 +49,28 @@ async function dumpN8nInstanceLogs($$) {
 	await $$`docker-compose logs n8n`;
 }
 
+function printUsage() {
+	const availableSetups = getAllN8nSetups();
+	console.log('Usage: zx runOnVm.mjs <n8n setup to use>');
+	console.log(`   eg: zx runOnVm.mjs ${availableSetups[0]}`);
+	console.log('');
+	console.log('Available setups:');
+	console.log(availableSetups.join(', '));
+}
+
+/**
+ * @returns {string[]}
+ */
+function getAllN8nSetups() {
+	return fs.readdirSync(paths.n8nSetupsDir);
+}
+
+function validateN8nSetup(givenSetup) {
+	const availableSetups = getAllN8nSetups();
+	if (!availableSetups.includes(givenSetup)) {
+		printUsage();
+		process.exit(1);
+	}
+}
+
 main();