From b18313f219589584a93fe22a0a2dfe6efc10c731 Mon Sep 17 00:00:00 2001 From: Tomi Turtiainen <10324676+tomi@users.noreply.github.com> Date: Mon, 9 Sep 2024 09:07:32 +0300 Subject: [PATCH] test: Add scaling n8n setup (multi-main) (#10644) --- .../n8nSetups/queue/docker-compose.yml | 8 +- .../n8nSetups/scaling/docker-compose.yml | 194 ++++++++++++++++++ .../scripts/n8nSetups/scaling/nginx.conf | 20 ++ .../scripts/n8nSetups/scaling/setup.mjs | 16 ++ packages/@n8n/benchmark/scripts/run.mjs | 5 + .../@n8n/benchmark/scripts/runForN8nSetup.mjs | 27 ++- .../@n8n/benchmark/scripts/runInCloud.mjs | 2 + .../@n8n/benchmark/scripts/runLocally.mjs | 2 + 8 files changed, 266 insertions(+), 8 deletions(-) create mode 100644 packages/@n8n/benchmark/scripts/n8nSetups/scaling/docker-compose.yml create mode 100644 packages/@n8n/benchmark/scripts/n8nSetups/scaling/nginx.conf create mode 100644 packages/@n8n/benchmark/scripts/n8nSetups/scaling/setup.mjs diff --git a/packages/@n8n/benchmark/scripts/n8nSetups/queue/docker-compose.yml b/packages/@n8n/benchmark/scripts/n8nSetups/queue/docker-compose.yml index 24995d91f..246f380a9 100644 --- a/packages/@n8n/benchmark/scripts/n8nSetups/queue/docker-compose.yml +++ b/packages/@n8n/benchmark/scripts/n8nSetups/queue/docker-compose.yml @@ -38,11 +38,12 @@ services: environment: - N8N_DIAGNOSTICS_ENABLED=false - N8N_USER_FOLDER=/n8n/worker1 - - N8N_ENCRYPTION_KEY=very-secret-encryption-key + - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY} # Queue mode config - EXECUTIONS_MODE=queue - QUEUE_BULL_REDIS_HOST=redis - QUEUE_HEALTH_CHECK_ACTIVE=true + - N8N_CONCURRENCY_PRODUCTION_LIMIT=10 # DB config - DB_TYPE=postgresdb - DB_POSTGRESDB_HOST=postgres @@ -67,11 +68,12 @@ services: environment: - N8N_DIAGNOSTICS_ENABLED=false - N8N_USER_FOLDER=/n8n/worker2 - - N8N_ENCRYPTION_KEY=very-secret-encryption-key + - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY} # Queue mode config - EXECUTIONS_MODE=queue - QUEUE_BULL_REDIS_HOST=redis - QUEUE_HEALTH_CHECK_ACTIVE=true + - N8N_CONCURRENCY_PRODUCTION_LIMIT=10 # DB config - DB_TYPE=postgresdb - DB_POSTGRESDB_HOST=postgres @@ -99,7 +101,7 @@ services: environment: - N8N_DIAGNOSTICS_ENABLED=false - N8N_USER_FOLDER=/n8n/main - - N8N_ENCRYPTION_KEY=very-secret-encryption-key + - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY} # Queue mode config - EXECUTIONS_MODE=queue - QUEUE_BULL_REDIS_HOST=redis diff --git a/packages/@n8n/benchmark/scripts/n8nSetups/scaling/docker-compose.yml b/packages/@n8n/benchmark/scripts/n8nSetups/scaling/docker-compose.yml new file mode 100644 index 000000000..d286e12bd --- /dev/null +++ b/packages/@n8n/benchmark/scripts/n8nSetups/scaling/docker-compose.yml @@ -0,0 +1,194 @@ +services: + mockapi: + image: wiremock/wiremock:3.9.1 + ports: + - '8088:8080' + volumes: + - ${MOCK_API_DATA_PATH}/mappings:/home/wiremock/mappings + + redis: + image: redis:6-alpine + restart: always + ports: + - 6379:6379 + healthcheck: + test: ['CMD', 'redis-cli', 'ping'] + interval: 1s + timeout: 3s + + postgres: + image: postgres:16 + restart: always + environment: + - POSTGRES_DB=n8n + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=password + - PGDATA=/var/lib/postgresql/data/pgdata + volumes: + - ${RUN_DIR}/postgres:/var/lib/postgresql/data + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U postgres'] + interval: 5s + timeout: 5s + retries: 10 + + n8n_worker1: + image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest} + environment: + - N8N_DIAGNOSTICS_ENABLED=false + - N8N_USER_FOLDER=/n8n/worker1 + - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY} + - N8N_LICENSE_CERT=${N8N_LICENSE_CERT} + - N8N_LICENSE_ACTIVATION_KEY=${N8N_LICENSE_ACTIVATION_KEY} + - N8N_LICENSE_TENANT_ID=${N8N_LICENSE_TENANT_ID} + # Scaling mode config + - EXECUTIONS_MODE=queue + - QUEUE_BULL_REDIS_HOST=redis + - QUEUE_HEALTH_CHECK_ACTIVE=true + - N8N_CONCURRENCY_PRODUCTION_LIMIT=10 + # DB config + - DB_TYPE=postgresdb + - DB_POSTGRESDB_HOST=postgres + - DB_POSTGRESDB_PASSWORD=password + command: worker + volumes: + - ${RUN_DIR}/n8n-worker1:/n8n + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: ['CMD-SHELL', 'wget --spider -q http://localhost:5678/healthz || exit 1'] + interval: 5s + timeout: 5s + retries: 10 + + n8n_worker2: + image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest} + environment: + - N8N_DIAGNOSTICS_ENABLED=false + - N8N_USER_FOLDER=/n8n/worker2 + - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY} + - N8N_LICENSE_CERT=${N8N_LICENSE_CERT} + - N8N_LICENSE_ACTIVATION_KEY=${N8N_LICENSE_ACTIVATION_KEY} + - N8N_LICENSE_TENANT_ID=${N8N_LICENSE_TENANT_ID} + # Scaling mode config + - EXECUTIONS_MODE=queue + - QUEUE_BULL_REDIS_HOST=redis + - QUEUE_HEALTH_CHECK_ACTIVE=true + - N8N_CONCURRENCY_PRODUCTION_LIMIT=10 + # DB config + - DB_TYPE=postgresdb + - DB_POSTGRESDB_HOST=postgres + - DB_POSTGRESDB_PASSWORD=password + command: worker + volumes: + - ${RUN_DIR}/n8n-worker2:/n8n + depends_on: + # We let the worker 1 start first so it can run the DB migrations + n8n_worker1: + condition: service_healthy + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: ['CMD-SHELL', 'wget --spider -q http://localhost:5678/healthz || exit 1'] + interval: 5s + timeout: 5s + retries: 10 + + n8n_main2: + image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest} + environment: + - N8N_DIAGNOSTICS_ENABLED=false + - N8N_USER_FOLDER=/n8n + - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY} + - N8N_LICENSE_CERT=${N8N_LICENSE_CERT} + - N8N_LICENSE_ACTIVATION_KEY=${N8N_LICENSE_ACTIVATION_KEY} + - N8N_LICENSE_TENANT_ID=${N8N_LICENSE_TENANT_ID} + # Scaling mode config + - EXECUTIONS_MODE=queue + - QUEUE_BULL_REDIS_HOST=redis + - N8N_MULTI_MAIN_SETUP_ENABLED=true + # DB config + - DB_TYPE=postgresdb + - DB_POSTGRESDB_HOST=postgres + - DB_POSTGRESDB_PASSWORD=password + volumes: + - ${RUN_DIR}/n8n-main2:/n8n + depends_on: + n8n_worker1: + condition: service_healthy + n8n_worker2: + condition: service_healthy + postgres: + condition: service_healthy + redis: + condition: service_healthy + mockapi: + condition: service_started + healthcheck: + test: ['CMD-SHELL', 'wget --spider -q http://n8n_main2:5678/healthz || exit 1'] + interval: 5s + timeout: 5s + retries: 10 + + n8n_main1: + image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest} + environment: + - N8N_DIAGNOSTICS_ENABLED=false + - N8N_USER_FOLDER=/n8n + - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY} + - N8N_LICENSE_CERT=${N8N_LICENSE_CERT} + - N8N_LICENSE_ACTIVATION_KEY=${N8N_LICENSE_ACTIVATION_KEY} + - N8N_LICENSE_TENANT_ID=${N8N_LICENSE_TENANT_ID} + # Scaling mode config + - EXECUTIONS_MODE=queue + - QUEUE_BULL_REDIS_HOST=redis + - N8N_MULTI_MAIN_SETUP_ENABLED=true + # DB config + - DB_TYPE=postgresdb + - DB_POSTGRESDB_HOST=postgres + - DB_POSTGRESDB_PASSWORD=password + + volumes: + - ${RUN_DIR}/n8n-main:/n8n + depends_on: + n8n_worker1: + condition: service_healthy + n8n_worker2: + condition: service_healthy + postgres: + condition: service_healthy + redis: + condition: service_healthy + mockapi: + condition: service_started + healthcheck: + test: ['CMD-SHELL', 'wget --spider -q http://n8n_main1:5678/healthz || exit 1'] + interval: 5s + timeout: 5s + retries: 10 + + # Load balancer that acts as an entry point for n8n + n8n: + image: nginx:latest + ports: + - '5678:80' + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf + depends_on: + n8n_main1: + condition: service_healthy + n8n_main2: + condition: service_healthy + + benchmark: + image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest} + depends_on: + - n8n + environment: + - N8N_BASE_URL=http://n8n:80 + - K6_API_TOKEN=${K6_API_TOKEN} diff --git a/packages/@n8n/benchmark/scripts/n8nSetups/scaling/nginx.conf b/packages/@n8n/benchmark/scripts/n8nSetups/scaling/nginx.conf new file mode 100644 index 000000000..1a42d4977 --- /dev/null +++ b/packages/@n8n/benchmark/scripts/n8nSetups/scaling/nginx.conf @@ -0,0 +1,20 @@ +events {} + +http { + upstream backend { + server n8n_main1:5678; + server n8n_main2:5678; + } + + server { + listen 80; + + location / { + proxy_pass http://backend; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } +} diff --git a/packages/@n8n/benchmark/scripts/n8nSetups/scaling/setup.mjs b/packages/@n8n/benchmark/scripts/n8nSetups/scaling/setup.mjs new file mode 100644 index 000000000..cb3300aae --- /dev/null +++ b/packages/@n8n/benchmark/scripts/n8nSetups/scaling/setup.mjs @@ -0,0 +1,16 @@ +#!/usr/bin/env zx + +import path from 'path'; +import { fs } from 'zx'; + +/** + * Creates the needed directories for the queue setup so their + * permissions get set correctly. + */ +export function setup({ runDir }) { + const neededDirs = ['n8n-worker1', 'n8n-worker2', 'n8n-main1', 'n8n_main2', 'postgres']; + + for (const dir of neededDirs) { + fs.ensureDirSync(path.join(runDir, dir)); + } +} diff --git a/packages/@n8n/benchmark/scripts/run.mjs b/packages/@n8n/benchmark/scripts/run.mjs index a276eee5f..f6f5da1e3 100755 --- a/packages/@n8n/benchmark/scripts/run.mjs +++ b/packages/@n8n/benchmark/scripts/run.mjs @@ -33,6 +33,7 @@ async function main() { benchmarkTag: config.benchmarkTag, isVerbose: config.isVerbose, k6ApiToken: config.k6ApiToken, + n8nLicenseCert: config.n8nLicenseCert, n8nTag: config.n8nTag, n8nSetupsToUse, }); @@ -41,6 +42,7 @@ async function main() { benchmarkTag: config.benchmarkTag, isVerbose: config.isVerbose, k6ApiToken: config.k6ApiToken, + n8nLicenseCert: config.n8nLicenseCert, n8nTag: config.n8nTag, runDir: config.runDir, n8nSetupsToUse, @@ -62,6 +64,7 @@ function readAvailableN8nSetups() { * @property {string} n8nTag * @property {string} benchmarkTag * @property {string} [k6ApiToken] + * @property {string} [n8nLicenseCert] * @property {string} [runDir] * * @returns {Promise} @@ -81,6 +84,7 @@ async function parseAndValidateConfig() { const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest'; const benchmarkTag = args.benchmarkTag || process.env.BENCHMARK_DOCKER_TAG || 'latest'; const k6ApiToken = args.k6ApiToken || process.env.K6_API_TOKEN || undefined; + const n8nLicenseCert = args.n8nLicenseCert || process.env.N8N_LICENSE_CERT || undefined; const runDir = args.runDir || undefined; const env = args.env || 'local'; @@ -96,6 +100,7 @@ async function parseAndValidateConfig() { n8nTag, benchmarkTag, k6ApiToken, + n8nLicenseCert, runDir, }; } diff --git a/packages/@n8n/benchmark/scripts/runForN8nSetup.mjs b/packages/@n8n/benchmark/scripts/runForN8nSetup.mjs index a46418ecc..b27968daa 100755 --- a/packages/@n8n/benchmark/scripts/runForN8nSetup.mjs +++ b/packages/@n8n/benchmark/scripts/runForN8nSetup.mjs @@ -12,6 +12,8 @@ const paths = { mockApiDataPath: path.join(__dirname, 'mockApi'), }; +const N8N_ENCRYPTION_KEY = 'very-secret-encryption-key'; + async function main() { const [n8nSetupToUse] = argv._; validateN8nSetup(n8nSetupToUse); @@ -22,6 +24,9 @@ async function main() { const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest'; const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined; const baseRunDir = argv.runDir || process.env.RUN_DIR || '/n8n'; + const n8nLicenseCert = argv.n8nLicenseCert || process.env.N8N_LICENSE_CERT || undefined; + const n8nLicenseActivationKey = process.env.N8N_LICENSE_ACTIVATION_KEY || ''; + const n8nLicenseTenantId = argv.n8nLicenseTenantId || process.env.N8N_LICENSE_TENANT_ID || ''; if (!fs.existsSync(baseRunDir)) { console.error( @@ -38,7 +43,12 @@ async function main() { cwd: composeFilePath, verbose: true, env: { + PATH: process.env.PATH, N8N_VERSION: n8nTag, + N8N_LICENSE_CERT: n8nLicenseCert, + N8N_LICENSE_ACTIVATION_KEY: n8nLicenseActivationKey, + N8N_LICENSE_TENANT_ID: n8nLicenseTenantId, + N8N_ENCRYPTION_KEY, BENCHMARK_VERSION: benchmarkTag, K6_API_TOKEN: k6ApiToken, RUN_DIR: runDir, @@ -59,17 +69,24 @@ async function main() { await dockerComposeClient.$('run', 'benchmark', 'run', `--scenarioNamePrefix=${n8nSetupToUse}`); } catch (error) { console.error('An error occurred while running the benchmarks:'); - console.error(error); + console.error(error.message); console.error(''); - await dumpN8nInstanceLogs(dockerComposeClient); + await printContainerStatus(dockerComposeClient); + console.error(''); + await dumpLogs(dockerComposeClient); } finally { await dockerComposeClient.$('down'); } } -async function dumpN8nInstanceLogs(dockerComposeClient) { - console.error('n8n instance logs:'); - await dockerComposeClient.$('logs', 'n8n'); +async function printContainerStatus(dockerComposeClient) { + console.error('Container statuses:'); + await dockerComposeClient.$('ps', '-a'); +} + +async function dumpLogs(dockerComposeClient) { + console.error('Container logs:'); + await dockerComposeClient.$('logs'); } function printUsage() { diff --git a/packages/@n8n/benchmark/scripts/runInCloud.mjs b/packages/@n8n/benchmark/scripts/runInCloud.mjs index 1c1b191ca..ba1c39241 100755 --- a/packages/@n8n/benchmark/scripts/runInCloud.mjs +++ b/packages/@n8n/benchmark/scripts/runInCloud.mjs @@ -29,6 +29,7 @@ import { TerraformClient } from './clients/terraformClient.mjs'; * @property {string} n8nTag * @property {string} benchmarkTag * @property {string} [k6ApiToken] + * @property {string} [n8nLicenseCert] * * @param {Config} config */ @@ -96,6 +97,7 @@ async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup n8nDockerTag: config.n8nTag, benchmarkDockerTag: config.benchmarkTag, k6ApiToken: config.k6ApiToken, + n8nLicenseCert: config.n8nLicenseCert, }; const flagsString = Object.entries(flags) diff --git a/packages/@n8n/benchmark/scripts/runLocally.mjs b/packages/@n8n/benchmark/scripts/runLocally.mjs index 8fcdcacfb..be3b83241 100755 --- a/packages/@n8n/benchmark/scripts/runLocally.mjs +++ b/packages/@n8n/benchmark/scripts/runLocally.mjs @@ -29,6 +29,7 @@ const paths = { * @property {string} benchmarkTag * @property {string} [runDir] * @property {string} [k6ApiToken] + * @property {string} [n8nLicenseCert] * * @param {Config} config */ @@ -51,6 +52,7 @@ export async function runLocally(config) { env: { ...process.env, K6_API_TOKEN: config.k6ApiToken, + N8N_LICENSE_CERT: config.n8nLicenseCert, }, })`npx ${runScriptPath} ${flags} ${n8nSetup}`; }