feat: External Secrets storage for credentials (#6477)

Github issue / Community forum post (link here to close automatically):

---------

Co-authored-by: Romain Minaud <romain.minaud@gmail.com>
Co-authored-by: Valya Bullions <valya@n8n.io>
Co-authored-by: Csaba Tuncsik <csaba@n8n.io>
Co-authored-by: Giulio Andreini <g.andreini@gmail.com>
Co-authored-by: Omar Ajoue <krynble@gmail.com>

packages/cli/src/ExternalSecrets/ExternalSecrets.controller.ee.ts

@@ -0,0 +1,102 @@
+import { Authorized, Get, Post, RestController } from '@/decorators';
+import { ExternalSecretsRequest } from '@/requests';
+import { NotFoundError } from '@/ResponseHelper';
+import { Response } from 'express';
+import { Service } from 'typedi';
+import { ProviderNotFoundError, ExternalSecretsService } from './ExternalSecrets.service.ee';
+
+@Service()
+@Authorized(['global', 'owner'])
+@RestController('/external-secrets')
+export class ExternalSecretsController {
+	constructor(private readonly secretsService: ExternalSecretsService) {}
+
+	@Get('/providers')
+	async getProviders() {
+		return this.secretsService.getProviders();
+	}
+
+	@Get('/providers/:provider')
+	async getProvider(req: ExternalSecretsRequest.GetProvider) {
+		const providerName = req.params.provider;
+		try {
+			return this.secretsService.getProvider(providerName);
+		} catch (e) {
+			if (e instanceof ProviderNotFoundError) {
+				throw new NotFoundError(`Could not find provider "${e.providerName}"`);
+			}
+			throw e;
+		}
+	}
+
+	@Post('/providers/:provider/test')
+	async testProviderSettings(req: ExternalSecretsRequest.TestProviderSettings, res: Response) {
+		const providerName = req.params.provider;
+		try {
+			const result = await this.secretsService.testProviderSettings(providerName, req.body);
+			if (result.success) {
+				res.statusCode = 200;
+			} else {
+				res.statusCode = 400;
+			}
+			return result;
+		} catch (e) {
+			if (e instanceof ProviderNotFoundError) {
+				throw new NotFoundError(`Could not find provider "${e.providerName}"`);
+			}
+			throw e;
+		}
+	}
+
+	@Post('/providers/:provider')
+	async setProviderSettings(req: ExternalSecretsRequest.SetProviderSettings) {
+		const providerName = req.params.provider;
+		try {
+			await this.secretsService.saveProviderSettings(providerName, req.body, req.user.id);
+		} catch (e) {
+			if (e instanceof ProviderNotFoundError) {
+				throw new NotFoundError(`Could not find provider "${e.providerName}"`);
+			}
+			throw e;
+		}
+		return {};
+	}
+
+	@Post('/providers/:provider/connect')
+	async setProviderConnected(req: ExternalSecretsRequest.SetProviderConnected) {
+		const providerName = req.params.provider;
+		try {
+			await this.secretsService.saveProviderConnected(providerName, req.body.connected);
+		} catch (e) {
+			if (e instanceof ProviderNotFoundError) {
+				throw new NotFoundError(`Could not find provider "${e.providerName}"`);
+			}
+			throw e;
+		}
+		return {};
+	}
+
+	@Post('/providers/:provider/update')
+	async updateProvider(req: ExternalSecretsRequest.UpdateProvider, res: Response) {
+		const providerName = req.params.provider;
+		try {
+			const resp = await this.secretsService.updateProvider(providerName);
+			if (resp) {
+				res.statusCode = 200;
+			} else {
+				res.statusCode = 400;
+			}
+			return { updated: resp };
+		} catch (e) {
+			if (e instanceof ProviderNotFoundError) {
+				throw new NotFoundError(`Could not find provider "${e.providerName}"`);
+			}
+			throw e;
+		}
+	}
+
+	@Get('/secrets')
+	getSecretNames() {
+		return this.secretsService.getAllSecrets();
+	}
+}

packages/cli/src/ExternalSecrets/ExternalSecrets.service.ee.ts

@@ -0,0 +1,154 @@
+import { CREDENTIAL_BLANKING_VALUE } from '@/constants';
+import type { SecretsProvider } from '@/Interfaces';
+import type { ExternalSecretsRequest } from '@/requests';
+import type { IDataObject } from 'n8n-workflow';
+import { deepCopy } from 'n8n-workflow';
+import Container, { Service } from 'typedi';
+import { ExternalSecretsManager } from './ExternalSecretsManager.ee';
+
+export class ProviderNotFoundError extends Error {
+	constructor(public providerName: string) {
+		super(undefined);
+	}
+}
+
+@Service()
+export class ExternalSecretsService {
+	getProvider(providerName: string): ExternalSecretsRequest.GetProviderResponse | null {
+		const providerAndSettings =
+			Container.get(ExternalSecretsManager).getProviderWithSettings(providerName);
+		if (!providerAndSettings) {
+			throw new ProviderNotFoundError(providerName);
+		}
+		const { provider, settings } = providerAndSettings;
+		return {
+			displayName: provider.displayName,
+			name: provider.name,
+			icon: provider.name,
+			state: provider.state,
+			connected: settings.connected,
+			connectedAt: settings.connectedAt,
+			properties: provider.properties,
+			data: this.redact(settings.settings, provider),
+		};
+	}
+
+	async getProviders() {
+		return Container.get(ExternalSecretsManager)
+			.getProvidersWithSettings()
+			.map(({ provider, settings }) => ({
+				displayName: provider.displayName,
+				name: provider.name,
+				icon: provider.name,
+				state: provider.state,
+				connected: !!settings.connected,
+				connectedAt: settings.connectedAt,
+				data: this.redact(settings.settings, provider),
+			}));
+	}
+
+	// Take data and replace all sensitive values with a sentinel value.
+	// This will replace password fields and oauth data.
+	redact(data: IDataObject, provider: SecretsProvider): IDataObject {
+		const copiedData = deepCopy(data || {});
+
+		const properties = provider.properties;
+
+		for (const dataKey of Object.keys(copiedData)) {
+			// The frontend only cares that this value isn't falsy.
+			if (dataKey === 'oauthTokenData') {
+				copiedData[dataKey] = CREDENTIAL_BLANKING_VALUE;
+				continue;
+			}
+			const prop = properties.find((v) => v.name === dataKey);
+			if (!prop) {
+				continue;
+			}
+
+			if (
+				prop.typeOptions?.password &&
+				(!(copiedData[dataKey] as string).startsWith('=') || prop.noDataExpression)
+			) {
+				copiedData[dataKey] = CREDENTIAL_BLANKING_VALUE;
+			}
+		}
+
+		return copiedData;
+	}
+
+	// eslint-disable-next-line @typescript-eslint/no-explicit-any
+	private unredactRestoreValues(unmerged: any, replacement: any) {
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+		for (const [key, value] of Object.entries(unmerged)) {
+			if (value === CREDENTIAL_BLANKING_VALUE) {
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
+				unmerged[key] = replacement[key];
+			} else if (
+				typeof value === 'object' &&
+				value !== null &&
+				key in replacement &&
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+				typeof replacement[key] === 'object' &&
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+				replacement[key] !== null
+			) {
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+				this.unredactRestoreValues(value, replacement[key]);
+			}
+		}
+	}
+
+	// Take unredacted data (probably from the DB) and merge it with
+	// redacted data to create an unredacted version.
+	unredact(redactedData: IDataObject, savedData: IDataObject): IDataObject {
+		// Replace any blank sentinel values with their saved version
+		const mergedData = deepCopy(redactedData ?? {});
+		this.unredactRestoreValues(mergedData, savedData);
+		return mergedData;
+	}
+
+	async saveProviderSettings(providerName: string, data: IDataObject, userId: string) {
+		const providerAndSettings =
+			Container.get(ExternalSecretsManager).getProviderWithSettings(providerName);
+		if (!providerAndSettings) {
+			throw new ProviderNotFoundError(providerName);
+		}
+		const { settings } = providerAndSettings;
+		const newData = this.unredact(data, settings.settings);
+		await Container.get(ExternalSecretsManager).setProviderSettings(providerName, newData, userId);
+	}
+
+	async saveProviderConnected(providerName: string, connected: boolean) {
+		const providerAndSettings =
+			Container.get(ExternalSecretsManager).getProviderWithSettings(providerName);
+		if (!providerAndSettings) {
+			throw new ProviderNotFoundError(providerName);
+		}
+		await Container.get(ExternalSecretsManager).setProviderConnected(providerName, connected);
+		return this.getProvider(providerName);
+	}
+
+	getAllSecrets(): Record<string, string[]> {
+		return Container.get(ExternalSecretsManager).getAllSecretNames();
+	}
+
+	async testProviderSettings(providerName: string, data: IDataObject) {
+		const providerAndSettings =
+			Container.get(ExternalSecretsManager).getProviderWithSettings(providerName);
+		if (!providerAndSettings) {
+			throw new ProviderNotFoundError(providerName);
+		}
+		const { settings } = providerAndSettings;
+		const newData = this.unredact(data, settings.settings);
+		return Container.get(ExternalSecretsManager).testProviderSettings(providerName, newData);
+	}
+
+	async updateProvider(providerName: string) {
+		const providerAndSettings =
+			Container.get(ExternalSecretsManager).getProviderWithSettings(providerName);
+		if (!providerAndSettings) {
+			throw new ProviderNotFoundError(providerName);
+		}
+		return Container.get(ExternalSecretsManager).updateProvider(providerName);
+	}
+}

packages/cli/src/ExternalSecrets/ExternalSecretsManager.ee.ts

@@ -0,0 +1,381 @@
+import { SettingsRepository } from '@/databases/repositories';
+import type {
+	ExternalSecretsSettings,
+	SecretsProvider,
+	SecretsProviderSettings,
+} from '@/Interfaces';
+
+import { UserSettings } from 'n8n-core';
+import Container, { Service } from 'typedi';
+
+import { AES, enc } from 'crypto-js';
+import { getLogger } from '@/Logger';
+
+import type { IDataObject } from 'n8n-workflow';
+import {
+	EXTERNAL_SECRETS_INITIAL_BACKOFF,
+	EXTERNAL_SECRETS_MAX_BACKOFF,
+	EXTERNAL_SECRETS_UPDATE_INTERVAL,
+} from './constants';
+import { License } from '@/License';
+import { InternalHooks } from '@/InternalHooks';
+import { ExternalSecretsProviders } from './ExternalSecretsProviders.ee';
+
+const logger = getLogger();
+
+@Service()
+export class ExternalSecretsManager {
+	private providers: Record<string, SecretsProvider> = {};
+
+	private initializingPromise?: Promise<void>;
+
+	private cachedSettings: ExternalSecretsSettings = {};
+
+	initialized = false;
+
+	updateInterval: NodeJS.Timer;
+
+	initRetryTimeouts: Record<string, NodeJS.Timer> = {};
+
+	constructor(
+		private settingsRepo: SettingsRepository,
+		private license: License,
+		private secretsProviders: ExternalSecretsProviders,
+	) {}
+
+	async init(): Promise<void> {
+		if (!this.initialized) {
+			if (!this.initializingPromise) {
+				this.initializingPromise = new Promise<void>(async (resolve) => {
+					await this.internalInit();
+					this.initialized = true;
+					resolve();
+					this.initializingPromise = undefined;
+					this.updateInterval = setInterval(
+						async () => this.updateSecrets(),
+						EXTERNAL_SECRETS_UPDATE_INTERVAL,
+					);
+				});
+			}
+			return this.initializingPromise;
+		}
+	}
+
+	shutdown() {
+		clearInterval(this.updateInterval);
+		Object.values(this.providers).forEach((p) => {
+			// Disregard any errors as we're shutting down anyway
+			void p.disconnect().catch(() => {});
+		});
+		Object.values(this.initRetryTimeouts).forEach((v) => clearTimeout(v));
+	}
+
+	private async getEncryptionKey(): Promise<string> {
+		return UserSettings.getEncryptionKey();
+	}
+
+	private decryptSecretsSettings(value: string, encryptionKey: string): ExternalSecretsSettings {
+		const decryptedData = AES.decrypt(value, encryptionKey);
+
+		try {
+			return JSON.parse(decryptedData.toString(enc.Utf8)) as ExternalSecretsSettings;
+		} catch (e) {
+			throw new Error(
+				'External Secrets Settings could not be decrypted. The likely reason is that a different "encryptionKey" was used to encrypt the data.',
+			);
+		}
+	}
+
+	private async getDecryptedSettings(
+		settingsRepo: SettingsRepository,
+	): Promise<ExternalSecretsSettings | null> {
+		const encryptedSettings = await settingsRepo.getEncryptedSecretsProviderSettings();
+		if (encryptedSettings === null) {
+			return null;
+		}
+		const encryptionKey = await this.getEncryptionKey();
+		return this.decryptSecretsSettings(encryptedSettings, encryptionKey);
+	}
+
+	private async internalInit() {
+		const settings = await this.getDecryptedSettings(this.settingsRepo);
+		if (!settings) {
+			return;
+		}
+		const providers: Array<SecretsProvider | null> = (
+			await Promise.allSettled(
+				Object.entries(settings).map(async ([name, providerSettings]) =>
+					this.initProvider(name, providerSettings),
+				),
+			)
+		).map((i) => (i.status === 'rejected' ? null : i.value));
+		this.providers = Object.fromEntries(
+			(providers.filter((p) => p !== null) as SecretsProvider[]).map((s) => [s.name, s]),
+		);
+		this.cachedSettings = settings;
+		await this.updateSecrets();
+	}
+
+	private async initProvider(
+		name: string,
+		providerSettings: SecretsProviderSettings,
+		currentBackoff = EXTERNAL_SECRETS_INITIAL_BACKOFF,
+	) {
+		const providerClass = this.secretsProviders.getProvider(name);
+		if (!providerClass) {
+			return null;
+		}
+		const provider: SecretsProvider = new providerClass();
+
+		try {
+			await provider.init(providerSettings);
+		} catch (e) {
+			logger.error(
+				`Error initializing secrets provider ${provider.displayName} (${provider.name}).`,
+			);
+			this.retryInitWithBackoff(name, currentBackoff);
+			return provider;
+		}
+
+		try {
+			if (providerSettings.connected) {
+				await provider.connect();
+			}
+		} catch (e) {
+			try {
+				await provider.disconnect();
+			} catch {}
+			logger.error(
+				`Error initializing secrets provider ${provider.displayName} (${provider.name}).`,
+			);
+			this.retryInitWithBackoff(name, currentBackoff);
+			return provider;
+		}
+
+		return provider;
+	}
+
+	private retryInitWithBackoff(name: string, currentBackoff: number) {
+		if (name in this.initRetryTimeouts) {
+			clearTimeout(this.initRetryTimeouts[name]);
+			delete this.initRetryTimeouts[name];
+		}
+		this.initRetryTimeouts[name] = setTimeout(() => {
+			delete this.initRetryTimeouts[name];
+			if (this.providers[name] && this.providers[name].state !== 'error') {
+				return;
+			}
+			void this.reloadProvider(name, Math.min(currentBackoff * 2, EXTERNAL_SECRETS_MAX_BACKOFF));
+		}, currentBackoff);
+	}
+
+	async updateSecrets() {
+		if (!this.license.isExternalSecretsEnabled()) {
+			return;
+		}
+		await Promise.allSettled(
+			Object.entries(this.providers).map(async ([k, p]) => {
+				try {
+					if (this.cachedSettings[k].connected && p.state === 'connected') {
+						await p.update();
+					}
+				} catch {
+					logger.error(`Error updating secrets provider ${p.displayName} (${p.name}).`);
+				}
+			}),
+		);
+	}
+
+	getProvider(provider: string): SecretsProvider | undefined {
+		return this.providers[provider];
+	}
+
+	hasProvider(provider: string): boolean {
+		return provider in this.providers;
+	}
+
+	getProviderNames(): string[] | undefined {
+		return Object.keys(this.providers);
+	}
+
+	getSecret(provider: string, name: string): IDataObject | undefined {
+		return this.getProvider(provider)?.getSecret(name);
+	}
+
+	hasSecret(provider: string, name: string): boolean {
+		return this.getProvider(provider)?.hasSecret(name) ?? false;
+	}
+
+	getSecretNames(provider: string): string[] | undefined {
+		return this.getProvider(provider)?.getSecretNames();
+	}
+
+	getAllSecretNames(): Record<string, string[]> {
+		return Object.fromEntries(
+			Object.keys(this.providers).map((provider) => [
+				provider,
+				this.getSecretNames(provider) ?? [],
+			]),
+		);
+	}
+
+	getProvidersWithSettings(): Array<{
+		provider: SecretsProvider;
+		settings: SecretsProviderSettings;
+	}> {
+		return Object.entries(this.secretsProviders.getAllProviders()).map(([k, c]) => ({
+			provider: this.getProvider(k) ?? new c(),
+			settings: this.cachedSettings[k] ?? {},
+		}));
+	}
+
+	getProviderWithSettings(provider: string):
+		| {
+				provider: SecretsProvider;
+				settings: SecretsProviderSettings;
+		  }
+		| undefined {
+		const providerConstructor = this.secretsProviders.getProvider(provider);
+		if (!providerConstructor) {
+			return undefined;
+		}
+		return {
+			provider: this.getProvider(provider) ?? new providerConstructor(),
+			settings: this.cachedSettings[provider] ?? {},
+		};
+	}
+
+	async reloadProvider(provider: string, backoff = EXTERNAL_SECRETS_INITIAL_BACKOFF) {
+		if (provider in this.providers) {
+			await this.providers[provider].disconnect();
+			delete this.providers[provider];
+		}
+		const newProvider = await this.initProvider(provider, this.cachedSettings[provider], backoff);
+		if (newProvider) {
+			this.providers[provider] = newProvider;
+		}
+	}
+
+	async setProviderSettings(provider: string, data: IDataObject, userId?: string) {
+		let isNewProvider = false;
+		let settings = await this.getDecryptedSettings(this.settingsRepo);
+		if (!settings) {
+			settings = {};
+		}
+		if (!(provider in settings)) {
+			isNewProvider = true;
+		}
+		settings[provider] = {
+			connected: settings[provider]?.connected ?? false,
+			connectedAt: settings[provider]?.connectedAt ?? new Date(),
+			settings: data,
+		};
+
+		await this.saveAndSetSettings(settings, this.settingsRepo);
+		this.cachedSettings = settings;
+		await this.reloadProvider(provider);
+
+		void this.trackProviderSave(provider, isNewProvider, userId);
+	}
+
+	async setProviderConnected(provider: string, connected: boolean) {
+		let settings = await this.getDecryptedSettings(this.settingsRepo);
+		if (!settings) {
+			settings = {};
+		}
+		settings[provider] = {
+			connected,
+			connectedAt: connected ? new Date() : settings[provider]?.connectedAt ?? null,
+			settings: settings[provider]?.settings ?? {},
+		};
+
+		await this.saveAndSetSettings(settings, this.settingsRepo);
+		this.cachedSettings = settings;
+		await this.reloadProvider(provider);
+		await this.updateSecrets();
+	}
+
+	private async trackProviderSave(vaultType: string, isNew: boolean, userId?: string) {
+		let testResult: [boolean] | [boolean, string] | undefined;
+		try {
+			testResult = await this.getProvider(vaultType)?.test();
+		} catch {}
+		void Container.get(InternalHooks).onExternalSecretsProviderSettingsSaved({
+			user_id: userId,
+			vault_type: vaultType,
+			is_new: isNew,
+			is_valid: testResult?.[0] ?? false,
+			error_message: testResult?.[1],
+		});
+	}
+
+	encryptSecretsSettings(settings: ExternalSecretsSettings, encryptionKey: string): string {
+		return AES.encrypt(JSON.stringify(settings), encryptionKey).toString();
+	}
+
+	async saveAndSetSettings(settings: ExternalSecretsSettings, settingsRepo: SettingsRepository) {
+		const encryptionKey = await this.getEncryptionKey();
+		const encryptedSettings = this.encryptSecretsSettings(settings, encryptionKey);
+		await settingsRepo.saveEncryptedSecretsProviderSettings(encryptedSettings);
+	}
+
+	async testProviderSettings(
+		provider: string,
+		data: IDataObject,
+	): Promise<{
+		success: boolean;
+		testState: 'connected' | 'tested' | 'error';
+		error?: string;
+	}> {
+		let testProvider: SecretsProvider | null = null;
+		try {
+			testProvider = await this.initProvider(provider, {
+				connected: true,
+				connectedAt: new Date(),
+				settings: data,
+			});
+			if (!testProvider) {
+				return {
+					success: false,
+					testState: 'error',
+				};
+			}
+			const [success, error] = await testProvider.test();
+			let testState: 'connected' | 'tested' | 'error' = 'error';
+			if (success && this.cachedSettings[provider]?.connected) {
+				testState = 'connected';
+			} else if (success) {
+				testState = 'tested';
+			}
+			return {
+				success,
+				testState,
+				error,
+			};
+		} catch {
+			return {
+				success: false,
+				testState: 'error',
+			};
+		} finally {
+			if (testProvider) {
+				await testProvider.disconnect();
+			}
+		}
+	}
+
+	async updateProvider(provider: string): Promise<boolean> {
+		if (!this.license.isExternalSecretsEnabled()) {
+			return false;
+		}
+		if (!this.providers[provider] || this.providers[provider].state !== 'connected') {
+			return false;
+		}
+		try {
+			await this.providers[provider].update();
+			return true;
+		} catch {
+			return false;
+		}
+	}
+}

packages/cli/src/ExternalSecrets/ExternalSecretsProviders.ee.ts

@@ -0,0 +1,24 @@
+import type { SecretsProvider } from '@/Interfaces';
+import { Service } from 'typedi';
+import { InfisicalProvider } from './providers/infisical';
+import { VaultProvider } from './providers/vault';
+
+@Service()
+export class ExternalSecretsProviders {
+	providers: Record<string, { new (): SecretsProvider }> = {
+		infisical: InfisicalProvider,
+		vault: VaultProvider,
+	};
+
+	getProvider(name: string): { new (): SecretsProvider } | null {
+		return this.providers[name] ?? null;
+	}
+
+	hasProvider(name: string) {
+		return name in this.providers;
+	}
+
+	getAllProviders() {
+		return this.providers;
+	}
+}

packages/cli/src/ExternalSecrets/constants.ts

@@ -0,0 +1,6 @@
+export const EXTERNAL_SECRETS_DB_KEY = 'feature.externalSecrets';
+export const EXTERNAL_SECRETS_UPDATE_INTERVAL = 5 * 60 * 1000;
+export const EXTERNAL_SECRETS_INITIAL_BACKOFF = 10 * 1000;
+export const EXTERNAL_SECRETS_MAX_BACKOFF = 5 * 60 * 1000;
+
+export const EXTERNAL_SECRETS_NAME_REGEX = /^[a-zA-Z0-9_]+$/;

packages/cli/src/ExternalSecrets/externalSecretsHelper.ee.ts

@@ -0,0 +1,7 @@
+import { License } from '@/License';
+import Container from 'typedi';
+
+export function isExternalSecretsEnabled() {
+	const license = Container.get(License);
+	return license.isExternalSecretsEnabled();
+}

packages/cli/src/ExternalSecrets/providers/infisical.ts

@@ -0,0 +1,153 @@
+import type { SecretsProvider, SecretsProviderSettings, SecretsProviderState } from '@/Interfaces';
+import InfisicalClient from 'infisical-node';
+import { populateClientWorkspaceConfigsHelper } from 'infisical-node/lib/helpers/key';
+import { getServiceTokenData } from 'infisical-node/lib/api/serviceTokenData';
+import type { IDataObject, INodeProperties } from 'n8n-workflow';
+import { EXTERNAL_SECRETS_NAME_REGEX } from '../constants';
+
+export interface InfisicalSettings {
+	token: string;
+	siteURL: string;
+	cacheTTL: number;
+	debug: boolean;
+}
+
+interface InfisicalSecret {
+	secretName: string;
+	secretValue?: string;
+}
+
+interface InfisicalServiceToken {
+	environment?: string;
+	scopes?: Array<{ environment: string; path: string }>;
+}
+
+export class InfisicalProvider implements SecretsProvider {
+	properties: INodeProperties[] = [
+		{
+			displayName:
+				'Need help filling out these fields? <a href="https://docs.n8n.io/external-secrets/#connect-n8n-to-your-secrets-store" target="_blank">Open docs</a>',
+			name: 'notice',
+			type: 'notice',
+			default: '',
+		},
+		{
+			displayName: 'Service Token',
+			name: 'token',
+			type: 'string',
+			hint: 'The Infisical Service Token with read access',
+			default: '',
+			required: true,
+			placeholder: 'e.g. st.64ae963e1874ea.374226a166439dce.39557e4a1b7bdd82',
+			noDataExpression: true,
+			typeOptions: { password: true },
+		},
+		{
+			displayName: 'Site URL',
+			name: 'siteURL',
+			type: 'string',
+			hint: "The absolute URL of the Infisical instance. Change it only if you're self-hosting Infisical.",
+			required: true,
+			noDataExpression: true,
+			placeholder: 'https://app.infisical.com',
+			default: 'https://app.infisical.com',
+		},
+	];
+
+	displayName = 'Infisical';
+
+	name = 'infisical';
+
+	state: SecretsProviderState = 'initializing';
+
+	private cachedSecrets: Record<string, string> = {};
+
+	private client: InfisicalClient;
+
+	private settings: InfisicalSettings;
+
+	private environment: string;
+
+	async init(settings: SecretsProviderSettings): Promise<void> {
+		this.settings = settings.settings as unknown as InfisicalSettings;
+	}
+
+	async update(): Promise<void> {
+		if (!this.client) {
+			throw new Error('Updated attempted on Infisical when initialization failed');
+		}
+		if (!(await this.test())[0]) {
+			throw new Error('Infisical provider test failed during update');
+		}
+		const secrets = (await this.client.getAllSecrets({
+			environment: this.environment,
+			path: '/',
+			attachToProcessEnv: false,
+			includeImports: true,
+		})) as InfisicalSecret[];
+		const newCache = Object.fromEntries(
+			secrets.map((s) => [s.secretName, s.secretValue]),
+		) as Record<string, string>;
+		if (Object.keys(newCache).length === 1 && '' in newCache) {
+			this.cachedSecrets = {};
+		} else {
+			this.cachedSecrets = newCache;
+		}
+	}
+
+	async connect(): Promise<void> {
+		this.client = new InfisicalClient(this.settings);
+		if ((await this.test())[0]) {
+			try {
+				this.environment = await this.getEnvironment();
+				this.state = 'connected';
+			} catch {
+				this.state = 'error';
+			}
+		} else {
+			this.state = 'error';
+		}
+	}
+
+	async getEnvironment(): Promise<string> {
+		const serviceTokenData = (await getServiceTokenData(
+			// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+			this.client.clientConfig,
+		)) as InfisicalServiceToken;
+		if (serviceTokenData.environment) {
+			return serviceTokenData.environment;
+		}
+		if (serviceTokenData.scopes) {
+			return serviceTokenData.scopes[0].environment;
+		}
+		throw new Error("Couldn't find environment for Infisical");
+	}
+
+	async test(): Promise<[boolean] | [boolean, string]> {
+		if (!this.client) {
+			return [false, 'Client not initialized'];
+		}
+		try {
+			await populateClientWorkspaceConfigsHelper(this.client.clientConfig);
+			return [true];
+		} catch (e) {
+			return [false];
+		}
+	}
+
+	async disconnect(): Promise<void> {
+		//
+	}
+
+	getSecret(name: string): IDataObject {
+		return this.cachedSecrets[name] as unknown as IDataObject;
+	}
+
+	getSecretNames(): string[] {
+		return Object.keys(this.cachedSecrets).filter((k) => EXTERNAL_SECRETS_NAME_REGEX.test(k));
+	}
+
+	hasSecret(name: string): boolean {
+		return name in this.cachedSecrets;
+	}
+}

packages/cli/src/ExternalSecrets/providers/vault.ts

@@ -0,0 +1,559 @@
+import type { SecretsProviderSettings, SecretsProviderState } from '@/Interfaces';
+import { SecretsProvider } from '@/Interfaces';
+import type { IDataObject, INodeProperties } from 'n8n-workflow';
+import type { AxiosInstance, AxiosResponse } from 'axios';
+import axios from 'axios';
+import { getLogger } from '@/Logger';
+import { EXTERNAL_SECRETS_NAME_REGEX } from '../constants';
+
+const logger = getLogger();
+
+type VaultAuthMethod = 'token' | 'usernameAndPassword' | 'appRole';
+
+interface VaultSettings {
+	url: string;
+	namespace?: string;
+	authMethod: VaultAuthMethod;
+
+	// Token
+	token: string;
+	renewToken: boolean;
+
+	// Username and Password
+	username: string;
+	password: string;
+
+	// AppRole
+	roleId: string;
+	secretId: string;
+}
+
+interface VaultResponse<T> {
+	data: T;
+}
+
+interface VaultTokenInfo {
+	accessor: string;
+	creation_time: number;
+	creation_ttl: number;
+	display_name: string;
+	entity_id: string;
+	expire_time: string | null;
+	explicit_max_ttl: number;
+	id: string;
+	issue_time: string;
+	meta: Record<string, string | number>;
+	num_uses: number;
+	orphan: boolean;
+	path: string;
+	policies: string[];
+	ttl: number;
+	renewable: boolean;
+	type: 'kv' | string;
+}
+
+interface VaultMount {
+	accessor: string;
+	config: Record<string, string | number | boolean | null>;
+	description: string;
+	external_entropy_access: boolean;
+	local: boolean;
+	options: Record<string, string | number | boolean | null>;
+	plugin_version: string;
+	running_plugin_version: string;
+	running_sha256: string;
+	seal_wrap: number;
+	type: string;
+	uuid: string;
+}
+
+interface VaultMountsResp {
+	[path: string]: VaultMount;
+}
+
+interface VaultUserPassLoginResp {
+	auth: {
+		client_token: string;
+	};
+}
+
+type VaultAppRoleResp = VaultUserPassLoginResp;
+
+interface VaultSecretList {
+	keys: string[];
+}
+
+export class VaultProvider extends SecretsProvider {
+	properties: INodeProperties[] = [
+		{
+			displayName:
+				'Need help filling out these fields? <a href="https://docs.n8n.io/external-secrets/#connect-n8n-to-your-secrets-store" target="_blank">Open docs</a>',
+			name: 'notice',
+			type: 'notice',
+			default: '',
+		},
+		{
+			displayName: 'Vault URL',
+			name: 'url',
+			type: 'string',
+			required: true,
+			noDataExpression: true,
+			placeholder: 'e.g. https://example.com/v1/',
+			default: '',
+		},
+		{
+			displayName: 'Vault Namespace (optional)',
+			name: 'namespace',
+			type: 'string',
+			hint: 'Leave blank if not using namespaces',
+			required: false,
+			noDataExpression: true,
+			placeholder: 'e.g. admin',
+			default: '',
+		},
+		{
+			displayName: 'Authentication Method',
+			name: 'authMethod',
+			type: 'options',
+			required: true,
+			noDataExpression: true,
+			options: [
+				{ name: 'Token', value: 'token' },
+				{ name: 'Username and Password', value: 'usernameAndPassword' },
+				{ name: 'AppRole', value: 'appRole' },
+			],
+			default: 'token',
+		},
+
+		// Token Auth
+		{
+			displayName: 'Token',
+			name: 'token',
+			type: 'string',
+			default: '',
+			required: true,
+			noDataExpression: true,
+			placeholder: 'e.g. hvs.2OCsZxZA6Z9lChbt0janOOZI',
+			typeOptions: { password: true },
+			displayOptions: {
+				show: {
+					authMethod: ['token'],
+				},
+			},
+		},
+		// {
+		// 	displayName: 'Renew Token',
+		// 	name: 'renewToken',
+		// 	description:
+		// 		'Try to renew Vault token. This will update the settings on this provider when doing so.',
+		// 	type: 'boolean',
+		// 	noDataExpression: true,
+		// 	default: true,
+		// 	displayOptions: {
+		// 		show: {
+		// 			authMethod: ['token'],
+		// 		},
+		// 	},
+		// },
+
+		// Username and Password
+		{
+			displayName: 'Username',
+			name: 'username',
+			type: 'string',
+			default: '',
+			required: true,
+			noDataExpression: true,
+			placeholder: 'Username',
+			displayOptions: {
+				show: {
+					authMethod: ['usernameAndPassword'],
+				},
+			},
+		},
+		{
+			displayName: 'Password',
+			name: 'password',
+			type: 'string',
+			default: '',
+			required: true,
+			noDataExpression: true,
+			placeholder: '***************',
+			typeOptions: { password: true },
+			displayOptions: {
+				show: {
+					authMethod: ['usernameAndPassword'],
+				},
+			},
+		},
+
+		// Username and Password
+		{
+			displayName: 'Role ID',
+			name: 'roleId',
+			type: 'string',
+			default: '',
+			required: true,
+			noDataExpression: true,
+			placeholder: '59d6d1ca-47bb-4e7e-a40b-8be3bc5a0ba8',
+			displayOptions: {
+				show: {
+					authMethod: ['appRole'],
+				},
+			},
+		},
+		{
+			displayName: 'Secret ID',
+			name: 'secretId',
+			type: 'string',
+			default: '',
+			required: true,
+			noDataExpression: true,
+			placeholder: '84896a0c-1347-aa90-a4f6-aca8b7558780',
+			typeOptions: { password: true },
+			displayOptions: {
+				show: {
+					authMethod: ['appRole'],
+				},
+			},
+		},
+	];
+
+	displayName = 'HashiCorp Vault';
+
+	name = 'vault';
+
+	state: SecretsProviderState = 'initializing';
+
+	private cachedSecrets: Record<string, IDataObject> = {};
+
+	private settings: VaultSettings;
+
+	#currentToken: string | null = null;
+
+	#tokenInfo: VaultTokenInfo | null = null;
+
+	#http: AxiosInstance;
+
+	private refreshTimeout: NodeJS.Timer | null;
+
+	private refreshAbort = new AbortController();
+
+	async init(settings: SecretsProviderSettings): Promise<void> {
+		this.settings = settings.settings as unknown as VaultSettings;
+
+		const baseURL = new URL(this.settings.url);
+
+		this.#http = axios.create({ baseURL: baseURL.toString() });
+		if (this.settings.namespace) {
+			this.#http.interceptors.request.use((config) => {
+				return {
+					...config,
+					// eslint-disable-next-line @typescript-eslint/naming-convention, @typescript-eslint/no-unsafe-assignment
+					headers: { ...config.headers, 'X-Vault-Namespace': this.settings.namespace },
+				};
+			});
+		}
+		this.#http.interceptors.request.use((config) => {
+			if (!this.#currentToken) {
+				return config;
+			}
+			// eslint-disable-next-line @typescript-eslint/naming-convention, @typescript-eslint/no-unsafe-assignment
+			return { ...config, headers: { ...config.headers, 'X-Vault-Token': this.#currentToken } };
+		});
+	}
+
+	async connect(): Promise<void> {
+		if (this.settings.authMethod === 'token') {
+			this.#currentToken = this.settings.token;
+		} else if (this.settings.authMethod === 'usernameAndPassword') {
+			try {
+				this.#currentToken = await this.authUsernameAndPassword(
+					this.settings.username,
+					this.settings.password,
+				);
+			} catch {
+				this.state = 'error';
+				logger.error('Failed to connect to Vault using Username and Password credentials.');
+				return;
+			}
+		} else if (this.settings.authMethod === 'appRole') {
+			try {
+				this.#currentToken = await this.authAppRole(this.settings.roleId, this.settings.secretId);
+			} catch {
+				this.state = 'error';
+				logger.error('Failed to connect to Vault using AppRole credentials.');
+				return;
+			}
+		}
+		try {
+			if (!(await this.test())[0]) {
+				this.state = 'error';
+			} else {
+				this.state = 'connected';
+
+				[this.#tokenInfo] = await this.getTokenInfo();
+				this.setupTokenRefresh();
+			}
+		} catch (e) {
+			this.state = 'error';
+			logger.error('Failed credentials test on Vault connect.');
+		}
+
+		try {
+			await this.update();
+		} catch {
+			logger.warn('Failed to update Vault secrets');
+		}
+	}
+
+	async disconnect(): Promise<void> {
+		if (this.refreshTimeout !== null) {
+			clearTimeout(this.refreshTimeout);
+		}
+		this.refreshAbort.abort();
+	}
+
+	private setupTokenRefresh() {
+		if (!this.#tokenInfo) {
+			return;
+		}
+		// Token never expires
+		if (this.#tokenInfo.expire_time === null) {
+			return;
+		}
+		// Token can't be renewed
+		if (!this.#tokenInfo.renewable) {
+			return;
+		}
+
+		const expireDate = new Date(this.#tokenInfo.expire_time);
+		setTimeout(this.tokenRefresh, (expireDate.valueOf() - Date.now()) / 2);
+	}
+
+	private tokenRefresh = async () => {
+		if (this.refreshAbort.signal.aborted) {
+			return;
+		}
+		try {
+			// We don't actually care about the result of this since it doesn't
+			// return an expire_time
+			await this.#http.post('auth/token/renew-self');
+
+			[this.#tokenInfo] = await this.getTokenInfo();
+
+			if (!this.#tokenInfo) {
+				logger.error('Failed to fetch token info during renewal. Cancelling all future renewals.');
+				return;
+			}
+
+			if (this.refreshAbort.signal.aborted) {
+				return;
+			}
+
+			this.setupTokenRefresh();
+		} catch {
+			logger.error('Failed to renew Vault token. Attempting to reconnect.');
+			void this.connect();
+		}
+	};
+
+	private async authUsernameAndPassword(
+		username: string,
+		password: string,
+	): Promise<string | null> {
+		try {
+			const resp = await this.#http.request<VaultUserPassLoginResp>({
+				method: 'POST',
+				url: `auth/userpass/login/${username}`,
+				responseType: 'json',
+				data: { password },
+			});
+
+			return resp.data.auth.client_token;
+		} catch {
+			return null;
+		}
+	}
+
+	private async authAppRole(roleId: string, secretId: string): Promise<string | null> {
+		try {
+			const resp = await this.#http.request<VaultAppRoleResp>({
+				method: 'POST',
+				url: 'auth/approle/login',
+				responseType: 'json',
+				data: { role_id: roleId, secret_id: secretId },
+			});
+
+			return resp.data.auth.client_token;
+		} catch (e) {
+			return null;
+		}
+	}
+
+	private async getTokenInfo(): Promise<[VaultTokenInfo | null, AxiosResponse]> {
+		const resp = await this.#http.request<VaultResponse<VaultTokenInfo>>({
+			method: 'GET',
+			url: 'auth/token/lookup-self',
+			responseType: 'json',
+			validateStatus: () => true,
+		});
+
+		if (resp.status !== 200 || !resp.data.data) {
+			return [null, resp];
+		}
+		return [resp.data.data, resp];
+	}
+
+	private async getKVSecrets(
+		mountPath: string,
+		kvVersion: string,
+		path: string,
+	): Promise<[string, IDataObject] | null> {
+		let listPath = mountPath;
+		if (kvVersion === '2') {
+			listPath += 'metadata/';
+		}
+		listPath += path;
+		let listResp: AxiosResponse<VaultResponse<VaultSecretList>>;
+		try {
+			listResp = await this.#http.request<VaultResponse<VaultSecretList>>({
+				url: listPath,
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any
+				method: 'LIST' as any,
+			});
+		} catch {
+			return null;
+		}
+		const data = Object.fromEntries(
+			(
+				await Promise.allSettled(
+					listResp.data.data.keys.map(async (key): Promise<[string, IDataObject] | null> => {
+						if (key.endsWith('/')) {
+							return this.getKVSecrets(mountPath, kvVersion, path + key);
+						}
+						let secretPath = mountPath;
+						if (kvVersion === '2') {
+							secretPath += 'data/';
+						}
+						secretPath += path + key;
+						try {
+							const secretResp = await this.#http.get<VaultResponse<IDataObject>>(secretPath);
+							return [
+								key,
+								kvVersion === '2'
+									? (secretResp.data.data.data as IDataObject)
+									: secretResp.data.data,
+							];
+						} catch {
+							return null;
+						}
+					}),
+				)
+			)
+				.map((i) => (i.status === 'rejected' ? null : i.value))
+				.filter((v) => v !== null) as Array<[string, IDataObject]>,
+		);
+		const name = path.substring(0, path.length - 1);
+		return [name, data];
+	}
+
+	async update(): Promise<void> {
+		const mounts = await this.#http.get<VaultResponse<VaultMountsResp>>('sys/mounts');
+
+		const kvs = Object.entries(mounts.data.data).filter(([, v]) => v.type === 'kv');
+
+		const secrets = Object.fromEntries(
+			(
+				await Promise.all(
+					kvs.map(async ([basePath, data]): Promise<[string, IDataObject] | null> => {
+						const value = await this.getKVSecrets(basePath, data.options.version as string, '');
+						if (value === null) {
+							return null;
+						}
+						return [basePath.substring(0, basePath.length - 1), value[1]];
+					}),
+				)
+			).filter((v) => v !== null) as Array<[string, IDataObject]>,
+		);
+		this.cachedSecrets = secrets;
+	}
+
+	async test(): Promise<[boolean] | [boolean, string]> {
+		try {
+			const [token, tokenResp] = await this.getTokenInfo();
+
+			if (token === null) {
+				if (tokenResp.status === 404) {
+					return [false, 'Could not find auth path. Try adding /v1/ to the end of your base URL.'];
+				}
+				return [false, 'Invalid credentials'];
+			}
+
+			const resp = await this.#http.request<VaultResponse<VaultTokenInfo>>({
+				method: 'GET',
+				url: 'sys/mounts',
+				responseType: 'json',
+				validateStatus: () => true,
+			});
+
+			if (resp.status === 403) {
+				return [
+					false,
+					"Couldn't list mounts. Please give these credentials 'read' access to sys/mounts.",
+				];
+			} else if (resp.status !== 200) {
+				return [
+					false,
+					"Couldn't list mounts but wasn't a permissions issue. Please consult your Vault admin.",
+				];
+			}
+
+			return [true];
+		} catch (e) {
+			if (axios.isAxiosError(e)) {
+				if (e.code === 'ECONNREFUSED') {
+					return [
+						false,
+						'Connection refused. Please check the host and port of the server are correct.',
+					];
+				}
+			}
+
+			return [false];
+		}
+	}
+
+	getSecret(name: string): IDataObject {
+		return this.cachedSecrets[name];
+	}
+
+	hasSecret(name: string): boolean {
+		return name in this.cachedSecrets;
+	}
+
+	getSecretNames(): string[] {
+		const getKeys = ([k, v]: [string, IDataObject]): string[] => {
+			if (!EXTERNAL_SECRETS_NAME_REGEX.test(k)) {
+				return [];
+			}
+			if (typeof v === 'object') {
+				const keys: string[] = [];
+				for (const key of Object.keys(v)) {
+					if (!EXTERNAL_SECRETS_NAME_REGEX.test(key)) {
+						continue;
+					}
+					const value = v[key];
+					if (typeof value === 'object' && value !== null) {
+						keys.push(...getKeys([key, value as IDataObject]).map((ok) => `${k}.${ok}`));
+					} else {
+						keys.push(`${k}.${key}`);
+					}
+				}
+				return keys;
+			}
+			return [k];
+		};
+		return Object.entries(this.cachedSecrets).flatMap(getKeys);
+	}
+}