feat: Benchmark env with run scripts (no-changelog) (#10477)

packages/@n8n/benchmark/scripts/destroyCloudEnv.mjs

@@ -0,0 +1,90 @@
+#!/usr/bin/env zx
+/**
+ * Script that deletes all resources created by the benchmark environment
+ * and that are older than 2 hours.
+ *
+ * Even tho the environment is provisioned using terraform, the terraform
+ * state is not persisted. Hence we can't use terraform to delete the resources.
+ * We could store the state to a storage account, but then we wouldn't be able
+ * to spin up new envs on-demand. Hence this design.
+ *
+ * Usage:
+ * 	 zx scripts/deleteCloudEnv.mjs
+ */
+// @ts-check
+import { $ } from 'zx';
+
+const EXPIRE_TIME_IN_H = 2;
+const EXPIRE_TIME_IN_MS = EXPIRE_TIME_IN_H * 60 * 60 * 1000;
+const RESOURCE_GROUP_NAME = 'n8n-benchmarking';
+
+async function main() {
+	const resourcesResult =
+		await $`az resource list --resource-group ${RESOURCE_GROUP_NAME} --query "[?tags.Id == 'N8nBenchmark'].{id:id, createdAt:tags.CreatedAt}" -o json`;
+
+	const resources = JSON.parse(resourcesResult.stdout);
+
+	const now = Date.now();
+
+	const resourcesToDelete = resources
+		.filter((resource) => {
+			if (resource.createdAt === undefined) {
+				return true;
+			}
+
+			const createdAt = new Date(resource.createdAt);
+			const resourceExpiredAt = createdAt.getTime() + EXPIRE_TIME_IN_MS;
+
+			return now > resourceExpiredAt;
+		})
+		.map((resource) => resource.id);
+
+	if (resourcesToDelete.length === 0) {
+		if (resources.length === 0) {
+			console.log('No resources found in the resource group.');
+		} else {
+			console.log(
+				`Found ${resources.length} resources in the resource group, but none are older than ${EXPIRE_TIME_IN_H} hours.`,
+			);
+		}
+
+		return;
+	}
+
+	await deleteResources(resourcesToDelete);
+}
+
+async function deleteResources(resourceIds) {
+	// We don't know the order in which resource should be deleted.
+	// Here's a poor person's approach to try deletion until all complete
+	const MAX_ITERATIONS = 100;
+	let i = 0;
+	const toDelete = [...resourceIds];
+
+	console.log(`Deleting ${resourceIds.length} resources...`);
+	while (toDelete.length > 0) {
+		const resourceId = toDelete.shift();
+		const deleted = await deleteById(resourceId);
+		if (!deleted) {
+			toDelete.push(resourceId);
+		}
+
+		if (i++ > MAX_ITERATIONS) {
+			console.log(
+				`Max iterations reached. Exiting. Could not delete ${toDelete.length} resources.`,
+			);
+			process.exit(1);
+		}
+	}
+}
+
+async function deleteById(id) {
+	try {
+		await $`az resource delete --ids ${id}`;
+		return true;
+	} catch (error) {
+		return false;
+	}
+}
+
+main();

packages/@n8n/benchmark/scripts/runInCloud.mjs

@@ -0,0 +1,185 @@
+#!/usr/bin/env zx
+/**
+ * Script to run benchmarks on the cloud benchmark environment.
+ * This script will:
+ * 	1. Provision a benchmark environment using Terraform.
+ * 	2. Run the benchmarks on the VM.
+ * 	3. Destroy the cloud environment.
+ *
+ * NOTE: Must be run in the root of the package.
+ *
+ * Usage:
+ * 	 zx scripts/runBenchmarksOnCloud.mjs [--debug] <n8n setup to use>
+ *
+ */
+// @ts-check
+import fs from 'fs';
+import minimist from 'minimist';
+import { $, sleep, tmpdir, which } from 'zx';
+import path from 'path';
+import { SshClient } from './sshClient.mjs';
+import { TerraformClient } from './terraformClient.mjs';
+
+/**
+ * @typedef {Object} BenchmarkEnv
+ * @property {string} vmName
+ */
+
+const RESOURCE_GROUP_NAME = 'n8n-benchmarking';
+
+const paths = {
+	n8nSetupsDir: path.join(path.resolve('scripts'), 'runOnVm', 'n8nSetups'),
+};
+
+async function main() {
+	const config = await parseAndValidateConfig();
+	await ensureDependencies();
+
+	console.log('Using n8n tag', config.n8nTag);
+	console.log('Using benchmark cli tag', config.benchmarkTag);
+
+	const terraformClient = new TerraformClient({
+		privateKeyPath: paths.privateKeyPath,
+		isVerbose: config.isVerbose,
+	});
+
+	try {
+		const benchmarkEnv = await terraformClient.provisionEnvironment();
+
+		await runBenchmarksOnVm(config, benchmarkEnv);
+	} catch (error) {
+		console.error('An error occurred while running the benchmarks:');
+		console.error(error);
+	} finally {
+		await terraformClient.destroyEnvironment();
+	}
+}
+
+async function ensureDependencies() {
+	await which('terraform');
+	await which('az');
+}
+
+/**
+ *
+ * @param {Config} config
+ * @param {BenchmarkEnv} benchmarkEnv
+ */
+async function runBenchmarksOnVm(config, benchmarkEnv) {
+	console.log(`Setting up the environment for ${config.n8nSetupToUse}...`);
+
+	const sshClient = new SshClient({
+		vmName: benchmarkEnv.vmName,
+		resourceGroupName: RESOURCE_GROUP_NAME,
+		verbose: config.isVerbose,
+	});
+
+	await ensureVmIsReachable(sshClient);
+
+	const scriptsDir = await transferScriptsToVm(sshClient);
+
+	// Bootstrap the environment with dependencies
+	console.log('Running bootstrap script...');
+	const bootstrapScriptPath = path.join(scriptsDir, 'bootstrap.sh');
+	await sshClient.ssh(`chmod a+x ${bootstrapScriptPath} && ${bootstrapScriptPath}`);
+
+	// Give some time for the VM to be ready
+	await sleep(1000);
+
+	console.log('Running benchmarks...');
+	const runScriptPath = path.join(scriptsDir, 'runOnVm.mjs');
+	await sshClient.ssh(
+		`npx zx ${runScriptPath} --n8nDockerTag=${config.n8nTag} --benchmarkDockerTag=${config.benchmarkTag} ${config.n8nSetupToUse}`,
+		{
+			// Test run should always log its output
+			verbose: true,
+		},
+	);
+}
+
+async function ensureVmIsReachable(sshClient) {
+	await sshClient.ssh('echo "VM is reachable"');
+}
+
+/**
+ * @returns Path where the scripts are located on the VM
+ */
+async function transferScriptsToVm(sshClient) {
+	await sshClient.ssh('rm -rf ~/n8n');
+
+	await sshClient.ssh('git clone --depth=0 https://github.com/n8n-io/n8n.git');
+
+	return '~/n8n/packages/@n8n/benchmark/scripts/runOnVm';
+}
+
+function readAvailableN8nSetups() {
+	const setups = fs.readdirSync(paths.n8nSetupsDir);
+
+	return setups;
+}
+
+/**
+ * @typedef {Object} Config
+ * @property {boolean} isVerbose
+ * @property {string} n8nSetupToUse
+ * @property {string} n8nTag
+ * @property {string} benchmarkTag
+ *
+ * @returns {Promise<Config>}
+ */
+async function parseAndValidateConfig() {
+	const args = minimist(process.argv.slice(2), {
+		boolean: ['debug'],
+	});
+
+	const n8nSetupToUse = await getAndValidateN8nSetup(args);
+	const isVerbose = args.debug || false;
+	const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest';
+	const benchmarkTag = args.benchmarkTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
+
+	return {
+		isVerbose,
+		n8nSetupToUse,
+		n8nTag,
+		benchmarkTag,
+	};
+}
+
+/**
+ * @param {minimist.ParsedArgs} args
+ */
+async function getAndValidateN8nSetup(args) {
+	// Last parameter is the n8n setup to use
+	const n8nSetupToUse = args._[args._.length - 1];
+
+	if (!n8nSetupToUse) {
+		printUsage();
+		process.exit(1);
+	}
+
+	const availableSetups = readAvailableN8nSetups();
+
+	if (!availableSetups.includes(n8nSetupToUse)) {
+		printUsage();
+		process.exit(1);
+	}
+
+	return n8nSetupToUse;
+}
+
+function printUsage() {
+	const availableSetups = readAvailableN8nSetups();
+
+	console.log('Usage: zx scripts/runInCloud.mjs <n8n setup name>');
+	console.log('   eg: zx scripts/runInCloud.mjs sqlite');
+	console.log('');
+	console.log('Options:');
+	console.log('  --debug              Enable verbose output');
+	console.log('  --n8nTag             Docker tag for n8n image. Default is latest');
+	console.log('  --benchmarkTag       Docker tag for benchmark cli image. Default is latest');
+	console.log('');
+	console.log('Available setups:');
+	console.log(`  ${availableSetups.join(', ')}`);
+}
+
+main().catch(console.error);

packages/@n8n/benchmark/scripts/runOnVm/bootstrap.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# Script to initialize the benchmark environment on a VM
+#
+
+set -euo pipefail;
+
+CURRENT_USER=$(whoami)
+
+# Mount the data disk
+if [ -d "/n8n" ]; then
+	echo "Data disk already mounted. Clearing it..."
+	rm -rf /n8n/*
+	rm -rf /n8n/.[!.]*
+else
+	sudo mkdir -p /n8n
+	sudo parted /dev/sdc --script mklabel gpt mkpart xfspart xfs 0% 100%
+	sudo mkfs.xfs /dev/sdc1
+	sudo partprobe /dev/sdc1
+	sudo mount /dev/sdc1 /n8n
+fi
+
+# Allow the current user to write to the data disk
+sudo chmod a+rw /n8n
+
+# Include nodejs v20 repository
+curl -fsSL https://deb.nodesource.com/setup_20.x -o nodesource_setup.sh
+sudo -E bash nodesource_setup.sh
+
+# Install docker, docker compose and nodejs
+sudo DEBIAN_FRONTEND=noninteractive apt-get update
+sudo DEBIAN_FRONTEND=noninteractive apt-get install -y docker.io docker-compose nodejs
+
+# Add the current user to the docker group
+sudo usermod -aG docker "$CURRENT_USER"
+
+# Install zx
+npm install zx

packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite/docker-compose.yml

@@ -0,0 +1,16 @@
+services:
+  n8n:
+    image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest}
+    environment:
+      - N8N_DIAGNOSTICS_ENABLED=false
+      - N8N_USER_FOLDER=/n8n
+    ports:
+      - 5678:5678
+    volumes:
+      - /n8n:/n8n
+  benchmark:
+    image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest}
+    depends_on:
+      - n8n
+    environment:
+      - N8N_BASE_URL=http://n8n:5678

packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs

@@ -0,0 +1,53 @@
+#!/usr/bin/env zx
+/**
+ * This script runs the benchmarks using a given docker compose setup
+ */
+
+import { $ } from 'zx';
+
+const [n8nSetupToUse] = argv._;
+
+if (!n8nSetupToUse) {
+	printUsage();
+	process.exit(1);
+}
+
+function printUsage() {
+	console.log('Usage: zx runOnVm.mjs <envName>');
+	console.log('   eg: zx runOnVm.mjs sqlite');
+}
+
+async function main() {
+	const composeFilePath = path.join(__dirname, 'n8nSetups', n8nSetupToUse);
+	const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest';
+	const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
+
+	const $$ = $({
+		cwd: composeFilePath,
+		verbose: true,
+		env: {
+			N8N_VERSION: n8nTag,
+			BENCHMARK_VERSION: benchmarkTag,
+		},
+	});
+
+	try {
+		await $$`docker-compose up -d n8n`;
+
+		await $$`docker-compose run benchmark run`;
+	} catch (error) {
+		console.error('An error occurred while running the benchmarks:');
+		console.error(error);
+		console.error('');
+		await dumpN8nInstanceLogs($$);
+	} finally {
+		await $$`docker-compose down`;
+	}
+}
+
+async function dumpN8nInstanceLogs($$) {
+	console.error('n8n instance logs:');
+	await $$`docker-compose logs n8n`;
+}
+
+main();

packages/@n8n/benchmark/scripts/sshClient.mjs

@@ -0,0 +1,28 @@
+// @ts-check
+import { $ } from 'zx';
+
+export class SshClient {
+	/**
+	 *
+	 * @param {{ vmName: string; resourceGroupName: string; verbose?: boolean }} param0
+	 */
+	constructor({ vmName, resourceGroupName, verbose = false }) {
+		this.vmName = vmName;
+		this.resourceGroupName = resourceGroupName;
+		this.verbose = verbose;
+
+		this.$$ = $({
+			verbose,
+		});
+	}
+
+	/**
+	 * @param {string} command
+	 * @param {{ verbose?: boolean }} [options]
+	 */
+	async ssh(command, options = {}) {
+		const $$ = options?.verbose ? $({ verbose: true }) : this.$$;
+
+		await $$`az ssh vm -n ${this.vmName} -g ${this.resourceGroupName} --yes -- -o StrictHostKeyChecking=accept-new ${command}`;
+	}
+}

packages/@n8n/benchmark/scripts/terraformClient.mjs

@@ -0,0 +1,53 @@
+// @ts-check
+
+import path from 'path';
+import { $, fs } from 'zx';
+
+const paths = {
+	infraCodeDir: path.resolve('infra'),
+	terraformStateFile: path.join(path.resolve('infra'), 'terraform.tfstate'),
+};
+
+export class TerraformClient {
+	constructor({ privateKeyPath, isVerbose = false }) {
+		this.privateKeyPath = privateKeyPath;
+		this.isVerbose = isVerbose;
+		this.$$ = $({
+			cwd: paths.infraCodeDir,
+			verbose: isVerbose,
+		});
+	}
+
+	/**
+	 * @typedef {Object} BenchmarkEnv
+	 * @property {string} vmName
+	 *
+	 * @returns {Promise<BenchmarkEnv>}
+	 */
+	async provisionEnvironment() {
+		console.log('Provisioning cloud environment...');
+
+		await this.$$`terraform init`;
+		await this.$$`terraform apply -input=false -auto-approve`;
+
+		return {
+			vmName: await this.getTerraformOutput('vm_name'),
+		};
+	}
+
+	async destroyEnvironment() {
+		if (!fs.existsSync(paths.terraformStateFile)) {
+			console.log('No cloud environment to destroy. Skipping...');
+			return;
+		}
+
+		console.log('Destroying cloud environment...');
+
+		await this.$$`terraform destroy -input=false -auto-approve`;
+	}
+
+	async getTerraformOutput(key) {
+		const output = await this.$$`terraform output -raw ${key}`;
+		return output.stdout.trim();
+	}
+}