feat: External Secrets storage for credentials (#6477)

Github issue / Community forum post (link here to close automatically):

---------

Co-authored-by: Romain Minaud <romain.minaud@gmail.com>
Co-authored-by: Valya Bullions <valya@n8n.io>
Co-authored-by: Csaba Tuncsik <csaba@n8n.io>
Co-authored-by: Giulio Andreini <g.andreini@gmail.com>
Co-authored-by: Omar Ajoue <krynble@gmail.com>
This commit is contained in:
Alex Grozav
2023-08-25 11:33:46 +03:00
committed by GitHub
parent c833078c87
commit ed927d34b2
89 changed files with 4164 additions and 57 deletions

View File

@@ -0,0 +1,370 @@
import type { SuperAgentTest } from 'supertest';
import { License } from '@/License';
import * as testDb from '../shared/testDb';
import * as utils from '../shared/utils/';
import type { ExternalSecretsSettings, SecretsProviderState } from '@/Interfaces';
import { UserSettings } from 'n8n-core';
import { SettingsRepository } from '@/databases/repositories/settings.repository';
import Container from 'typedi';
import { AES, enc } from 'crypto-js';
import { ExternalSecretsProviders } from '@/ExternalSecrets/ExternalSecretsProviders.ee';
import {
DummyProvider,
FailedProvider,
MockProviders,
TestFailProvider,
} from '../../shared/ExternalSecrets/utils';
import config from '@/config';
import { ExternalSecretsManager } from '@/ExternalSecrets/ExternalSecretsManager.ee';
import { CREDENTIAL_BLANKING_VALUE } from '@/constants';
import type { IDataObject } from 'n8n-workflow';
let authOwnerAgent: SuperAgentTest;
let authMemberAgent: SuperAgentTest;
const licenseLike = utils.mockInstance(License, {
isExternalSecretsEnabled: jest.fn().mockReturnValue(true),
isWithinUsersLimit: jest.fn().mockReturnValue(true),
});
const mockProvidersInstance = new MockProviders();
let providersMock: ExternalSecretsProviders = utils.mockInstance(
ExternalSecretsProviders,
mockProvidersInstance,
);
const testServer = utils.setupTestServer({ endpointGroups: ['externalSecrets'] });
const connectedDate = '2023-08-01T12:32:29.000Z';
async function setExternalSecretsSettings(settings: ExternalSecretsSettings) {
const encryptionKey = await UserSettings.getEncryptionKey();
return Container.get(SettingsRepository).saveEncryptedSecretsProviderSettings(
AES.encrypt(JSON.stringify(settings), encryptionKey).toString(),
);
}
async function getExternalSecretsSettings(): Promise<ExternalSecretsSettings | null> {
const encryptionKey = await UserSettings.getEncryptionKey();
const encSettings = await Container.get(SettingsRepository).getEncryptedSecretsProviderSettings();
if (encSettings === null) {
return null;
}
return JSON.parse(AES.decrypt(encSettings, encryptionKey).toString(enc.Utf8));
}
const resetManager = async () => {
Container.get(ExternalSecretsManager).shutdown();
Container.set(
ExternalSecretsManager,
new ExternalSecretsManager(
Container.get(SettingsRepository),
licenseLike,
mockProvidersInstance,
),
);
await Container.get(ExternalSecretsManager).init();
};
const getDummyProviderData = ({
data,
includeProperties,
connected,
state,
connectedAt,
displayName,
}: {
data?: IDataObject;
includeProperties?: boolean;
connected?: boolean;
state?: SecretsProviderState;
connectedAt?: string | null;
displayName?: string;
} = {}) => {
const dummy: IDataObject = {
connected: connected ?? true,
connectedAt: connectedAt === undefined ? connectedDate : connectedAt,
data: data ?? {},
name: 'dummy',
displayName: displayName ?? 'Dummy Provider',
icon: 'dummy',
state: state ?? 'connected',
};
if (includeProperties) {
dummy.properties = new DummyProvider().properties;
}
return dummy;
};
beforeAll(async () => {
await utils.initEncryptionKey();
const owner = await testDb.createOwner();
authOwnerAgent = testServer.authAgentFor(owner);
const member = await testDb.createUser();
authMemberAgent = testServer.authAgentFor(member);
config.set('userManagement.isInstanceOwnerSetUp', true);
});
beforeEach(async () => {
licenseLike.isExternalSecretsEnabled.mockReturnValue(true);
mockProvidersInstance.setProviders({
dummy: DummyProvider,
});
await setExternalSecretsSettings({
dummy: {
connected: true,
connectedAt: new Date(connectedDate),
settings: {},
},
});
await resetManager();
});
afterEach(async () => {
Container.get(ExternalSecretsManager).shutdown();
});
describe('GET /external-secrets/providers', () => {
test('can retrieve providers as owner', async () => {
const resp = await authOwnerAgent.get('/external-secrets/providers');
expect(resp.body).toEqual({
data: [getDummyProviderData()],
});
});
test('can not retrieve providers as non-owner', async () => {
const resp = await authMemberAgent.get('/external-secrets/providers');
expect(resp.status).toBe(403);
});
test('does obscure passwords', async () => {
await setExternalSecretsSettings({
dummy: {
connected: true,
connectedAt: new Date(connectedDate),
settings: {
username: 'testuser',
password: 'testpass',
},
},
});
await resetManager();
const resp = await authOwnerAgent.get('/external-secrets/providers');
expect(resp.body).toEqual({
data: [
getDummyProviderData({
data: {
username: 'testuser',
password: CREDENTIAL_BLANKING_VALUE,
},
}),
],
});
});
});
describe('GET /external-secrets/providers/:provider', () => {
test('can retrieve provider as owner', async () => {
const resp = await authOwnerAgent.get('/external-secrets/providers/dummy');
expect(resp.body.data).toEqual(getDummyProviderData({ includeProperties: true }));
});
test('can not retrieve provider as non-owner', async () => {
const resp = await authMemberAgent.get('/external-secrets/providers/dummy');
expect(resp.status).toBe(403);
});
test('does obscure passwords', async () => {
await setExternalSecretsSettings({
dummy: {
connected: true,
connectedAt: new Date(connectedDate),
settings: {
username: 'testuser',
password: 'testpass',
},
},
});
await resetManager();
const resp = await authOwnerAgent.get('/external-secrets/providers/dummy');
expect(resp.body.data).toEqual(
getDummyProviderData({
data: {
username: 'testuser',
password: CREDENTIAL_BLANKING_VALUE,
},
includeProperties: true,
}),
);
});
});
describe('POST /external-secrets/providers/:provider', () => {
test('can update provider settings', async () => {
const testData = {
username: 'testuser',
other: 'testother',
};
const resp = await authOwnerAgent.post('/external-secrets/providers/dummy').send(testData);
expect(resp.status).toBe(200);
const confirmResp = await authOwnerAgent.get('/external-secrets/providers/dummy');
expect(confirmResp.body.data).toEqual(
getDummyProviderData({ data: testData, includeProperties: true }),
);
});
test('can update provider settings with blanking value', async () => {
await setExternalSecretsSettings({
dummy: {
connected: true,
connectedAt: new Date(connectedDate),
settings: {
username: 'testuser',
password: 'testpass',
},
},
});
await resetManager();
const testData = {
username: 'newuser',
password: CREDENTIAL_BLANKING_VALUE,
};
const resp = await authOwnerAgent.post('/external-secrets/providers/dummy').send(testData);
expect(resp.status).toBe(200);
const confirmResp = await authOwnerAgent.get('/external-secrets/providers/dummy');
expect((await getExternalSecretsSettings())?.dummy.settings).toEqual({
username: 'newuser',
password: 'testpass',
});
});
});
describe('POST /external-secrets/providers/:provider/connect', () => {
test('can change provider connected state', async () => {
const testData = {
connected: false,
};
const resp = await authOwnerAgent
.post('/external-secrets/providers/dummy/connect')
.send(testData);
expect(resp.status).toBe(200);
const confirmResp = await authOwnerAgent.get('/external-secrets/providers/dummy');
expect(confirmResp.body.data).toEqual(
getDummyProviderData({
includeProperties: true,
connected: false,
state: 'initializing',
}),
);
});
});
describe('POST /external-secrets/providers/:provider/test', () => {
test('can test provider', async () => {
const testData = {
username: 'testuser',
other: 'testother',
};
const resp = await authOwnerAgent.post('/external-secrets/providers/dummy/test').send(testData);
expect(resp.status).toBe(200);
expect(resp.body.data.success).toBe(true);
expect(resp.body.data.testState).toBe('connected');
});
test('can test provider fail', async () => {
mockProvidersInstance.setProviders({
dummy: TestFailProvider,
});
await resetManager();
const testData = {
username: 'testuser',
other: 'testother',
};
const resp = await authOwnerAgent.post('/external-secrets/providers/dummy/test').send(testData);
expect(resp.status).toBe(400);
expect(resp.body.data.success).toBe(false);
expect(resp.body.data.testState).toBe('error');
});
});
describe('POST /external-secrets/providers/:provider/update', () => {
test('can update provider', async () => {
const updateSpy = jest.spyOn(
Container.get(ExternalSecretsManager).getProvider('dummy')!,
'update',
);
const resp = await authOwnerAgent.post('/external-secrets/providers/dummy/update');
expect(resp.status).toBe(200);
expect(resp.body.data).toEqual({ updated: true });
expect(updateSpy).toBeCalled();
});
test('can not update errored provider', async () => {
mockProvidersInstance.setProviders({
dummy: FailedProvider,
});
await resetManager();
const updateSpy = jest.spyOn(
Container.get(ExternalSecretsManager).getProvider('dummy')!,
'update',
);
const resp = await authOwnerAgent.post('/external-secrets/providers/dummy/update');
expect(resp.status).toBe(400);
expect(resp.body.data).toEqual({ updated: false });
expect(updateSpy).not.toBeCalled();
});
test('can not update provider without a valid license', async () => {
const updateSpy = jest.spyOn(
Container.get(ExternalSecretsManager).getProvider('dummy')!,
'update',
);
licenseLike.isExternalSecretsEnabled.mockReturnValue(false);
const resp = await authOwnerAgent.post('/external-secrets/providers/dummy/update');
expect(resp.status).toBe(400);
expect(resp.body.data).toEqual({ updated: false });
expect(updateSpy).not.toBeCalled();
});
});
describe('GET /external-secrets/secrets', () => {
test('can get secret names as owner', async () => {
const resp = await authOwnerAgent.get('/external-secrets/secrets');
expect(resp.status).toBe(200);
expect(resp.body.data).toEqual({
dummy: ['test1', 'test2'],
});
});
test('can not get secret names as non-owner', async () => {
const resp = await authMemberAgent.get('/external-secrets/secrets');
expect(resp.status).toBe(403);
expect(resp.body.data).not.toEqual({
dummy: ['test1', 'test2'],
});
});
});

View File

@@ -26,6 +26,7 @@ export type EndpointGroup =
| 'license'
| 'variables'
| 'tags'
| 'externalSecrets'
| 'mfa'
| 'metrics';

View File

@@ -50,6 +50,7 @@ import * as testDb from '../../shared/testDb';
import { AUTHLESS_ENDPOINTS, PUBLIC_API_REST_PATH_SEGMENT, REST_PATH_SEGMENT } from '../constants';
import type { EndpointGroup, SetupProps, TestServer } from '../types';
import { mockInstance } from './mocking';
import { ExternalSecretsController } from '@/ExternalSecrets/ExternalSecrets.controller.ee';
import { MfaService } from '@/Mfa/mfa.service';
import { TOTPService } from '@/Mfa/totp.service';
import { UserSettings } from 'n8n-core';
@@ -285,6 +286,9 @@ export const setupTestServer = ({
case 'tags':
registerController(app, config, Container.get(TagsController));
break;
case 'externalSecrets':
registerController(app, config, Container.get(ExternalSecretsController));
break;
}
}
}

View File

@@ -0,0 +1,215 @@
import { SecretsProvider } from '@/Interfaces';
import type { SecretsProviderSettings, SecretsProviderState } from '@/Interfaces';
import type { IDataObject, INodeProperties } from 'n8n-workflow';
export class MockProviders {
providers: Record<string, { new (): SecretsProvider }> = {
dummy: DummyProvider,
};
setProviders(providers: Record<string, { new (): SecretsProvider }>) {
this.providers = providers;
}
getProvider(name: string): { new (): SecretsProvider } | null {
return this.providers[name] ?? null;
}
hasProvider(name: string) {
return name in this.providers;
}
getAllProviders() {
return this.providers;
}
}
export class DummyProvider extends SecretsProvider {
properties: INodeProperties[] = [
{
name: 'username',
displayName: 'Username',
type: 'string',
default: '',
required: true,
},
{
name: 'other',
displayName: 'Other',
type: 'string',
default: '',
},
{
name: 'password',
displayName: 'Password',
type: 'string',
default: '',
typeOptions: {
password: true,
},
},
];
secrets: Record<string, string> = {};
displayName = 'Dummy Provider';
name = 'dummy';
state: SecretsProviderState = 'initializing';
_updateSecrets: Record<string, string> = {
test1: 'value1',
test2: 'value2',
};
async init(settings: SecretsProviderSettings<IDataObject>): Promise<void> {}
async connect(): Promise<void> {
this.state = 'connected';
}
async disconnect(): Promise<void> {}
async update(): Promise<void> {
this.secrets = this._updateSecrets;
}
async test(): Promise<[boolean] | [boolean, string]> {
return [true];
}
getSecret(name: string): IDataObject | undefined {
return this.secrets[name] as unknown as IDataObject | undefined;
}
hasSecret(name: string): boolean {
return name in this.secrets;
}
getSecretNames(): string[] {
return Object.keys(this.secrets);
}
}
export class ErrorProvider extends SecretsProvider {
secrets: Record<string, string> = {};
displayName = 'Error Provider';
name = 'dummy';
state: SecretsProviderState = 'initializing';
async init(settings: SecretsProviderSettings<IDataObject>): Promise<void> {
throw new Error();
}
async connect(): Promise<void> {
this.state = 'error';
throw new Error();
}
async disconnect(): Promise<void> {
throw new Error();
}
async update(): Promise<void> {
throw new Error();
}
async test(): Promise<[boolean] | [boolean, string]> {
throw new Error();
}
getSecret(name: string): IDataObject | undefined {
throw new Error();
}
hasSecret(name: string): boolean {
throw new Error();
}
getSecretNames(): string[] {
throw new Error();
}
}
export class FailedProvider extends SecretsProvider {
secrets: Record<string, string> = {};
displayName = 'Failed Provider';
name = 'dummy';
state: SecretsProviderState = 'initializing';
async init(settings: SecretsProviderSettings<IDataObject>): Promise<void> {}
async connect(): Promise<void> {
this.state = 'error';
}
async disconnect(): Promise<void> {}
async update(): Promise<void> {}
async test(): Promise<[boolean] | [boolean, string]> {
return [true];
}
getSecret(name: string): IDataObject | undefined {
return this.secrets[name] as unknown as IDataObject | undefined;
}
hasSecret(name: string): boolean {
return name in this.secrets;
}
getSecretNames(): string[] {
return Object.keys(this.secrets);
}
}
export class TestFailProvider extends SecretsProvider {
secrets: Record<string, string> = {};
displayName = 'Test Failed Provider';
name = 'dummy';
state: SecretsProviderState = 'initializing';
_updateSecrets: Record<string, string> = {
test1: 'value1',
test2: 'value2',
};
async init(settings: SecretsProviderSettings<IDataObject>): Promise<void> {}
async connect(): Promise<void> {
this.state = 'connected';
}
async disconnect(): Promise<void> {}
async update(): Promise<void> {
this.secrets = this._updateSecrets;
}
async test(): Promise<[boolean] | [boolean, string]> {
return [false];
}
getSecret(name: string): IDataObject | undefined {
return this.secrets[name] as unknown as IDataObject | undefined;
}
hasSecret(name: string): boolean {
return name in this.secrets;
}
getSecretNames(): string[] {
return Object.keys(this.secrets);
}
}

View File

@@ -24,6 +24,7 @@ import { mockInstance } from '../integration/shared/utils/';
import { Push } from '@/push';
import { ActiveExecutions } from '@/ActiveExecutions';
import { NodeTypes } from '@/NodeTypes';
import { SecretsHelper } from '@/SecretsHelpers';
import { WebhookService } from '@/services/webhook.service';
import { VariablesService } from '../../src/environments/variables/variables.service';
@@ -159,6 +160,7 @@ describe('ActiveWorkflowRunner', () => {
Container.set(LoadNodesAndCredentials, nodesAndCredentials);
Container.set(VariablesService, mockVariablesService);
mockInstance(Push);
mockInstance(SecretsHelper);
});
beforeEach(() => {

View File

@@ -0,0 +1,194 @@
import type { SettingsRepository } from '@/databases/repositories';
import type { ExternalSecretsSettings } from '@/Interfaces';
import { License } from '@/License';
import { ExternalSecretsManager } from '@/ExternalSecrets/ExternalSecretsManager.ee';
import { ExternalSecretsProviders } from '@/ExternalSecrets/ExternalSecretsProviders.ee';
import { mock } from 'jest-mock-extended';
import { UserSettings } from 'n8n-core';
import Container from 'typedi';
import { mockInstance } from '../../integration/shared/utils';
import {
DummyProvider,
ErrorProvider,
FailedProvider,
MockProviders,
} from '../../shared/ExternalSecrets/utils';
import { AES, enc } from 'crypto-js';
import { InternalHooks } from '@/InternalHooks';
const connectedDate = '2023-08-01T12:32:29.000Z';
const encryptionKey = 'testkey';
let settings: string | null = null;
const mockProvidersInstance = new MockProviders();
const settingsRepo = mock<SettingsRepository>({
async getEncryptedSecretsProviderSettings() {
return settings;
},
async saveEncryptedSecretsProviderSettings(data) {
settings = data;
},
});
let licenseMock: License;
let providersMock: ExternalSecretsProviders;
let manager: ExternalSecretsManager | undefined;
const createMockSettings = (settings: ExternalSecretsSettings): string => {
return AES.encrypt(JSON.stringify(settings), encryptionKey).toString();
};
const decryptSettings = (settings: string) => {
return JSON.parse(AES.decrypt(settings ?? '', encryptionKey).toString(enc.Utf8));
};
describe('External Secrets Manager', () => {
beforeAll(() => {
jest
.spyOn(UserSettings, 'getEncryptionKey')
.mockReturnValue(new Promise((resolve) => resolve(encryptionKey)));
providersMock = mockInstance(ExternalSecretsProviders, mockProvidersInstance);
licenseMock = mockInstance(License, {
isExternalSecretsEnabled() {
return true;
},
});
mockInstance(InternalHooks);
});
beforeEach(() => {
mockProvidersInstance.setProviders({
dummy: DummyProvider,
});
settings = createMockSettings({
dummy: { connected: true, connectedAt: new Date(connectedDate), settings: {} },
});
Container.remove(ExternalSecretsManager);
});
afterEach(() => {
manager?.shutdown();
jest.useRealTimers();
});
test('should get secret', async () => {
manager = new ExternalSecretsManager(settingsRepo, licenseMock, providersMock);
await manager.init();
expect(manager.getSecret('dummy', 'test1')).toBe('value1');
});
test('should not throw errors during init', async () => {
mockProvidersInstance.setProviders({
dummy: ErrorProvider,
});
manager = new ExternalSecretsManager(settingsRepo, licenseMock, providersMock);
expect(async () => manager!.init()).not.toThrow();
});
test('should not throw errors during shutdown', async () => {
mockProvidersInstance.setProviders({
dummy: ErrorProvider,
});
manager = new ExternalSecretsManager(settingsRepo, licenseMock, providersMock);
await manager.init();
expect(() => manager!.shutdown()).not.toThrow();
manager = undefined;
});
test('should save provider settings', async () => {
manager = new ExternalSecretsManager(settingsRepo, licenseMock, providersMock);
const settingsSpy = jest.spyOn(settingsRepo, 'saveEncryptedSecretsProviderSettings');
await manager.init();
await manager.setProviderSettings('dummy', {
test: 'value',
});
expect(decryptSettings(settingsSpy.mock.calls[0][0])).toEqual({
dummy: {
connected: true,
connectedAt: connectedDate,
settings: {
test: 'value',
},
},
});
});
test('should call provider update functions on a timer', async () => {
jest.useFakeTimers();
manager = new ExternalSecretsManager(settingsRepo, licenseMock, providersMock);
await manager.init();
const updateSpy = jest.spyOn(manager.getProvider('dummy')!, 'update');
expect(updateSpy).toBeCalledTimes(0);
jest.runOnlyPendingTimers();
expect(updateSpy).toBeCalledTimes(1);
});
test('should not call provider update functions if the not licensed', async () => {
jest.useFakeTimers();
manager = new ExternalSecretsManager(
settingsRepo,
mock<License>({
isExternalSecretsEnabled() {
return false;
},
}),
providersMock,
);
await manager.init();
const updateSpy = jest.spyOn(manager.getProvider('dummy')!, 'update');
expect(updateSpy).toBeCalledTimes(0);
jest.runOnlyPendingTimers();
expect(updateSpy).toBeCalledTimes(0);
});
test('should not call provider update functions if the provider has an error', async () => {
jest.useFakeTimers();
mockProvidersInstance.setProviders({
dummy: FailedProvider,
});
manager = new ExternalSecretsManager(settingsRepo, licenseMock, providersMock);
await manager.init();
const updateSpy = jest.spyOn(manager.getProvider('dummy')!, 'update');
expect(updateSpy).toBeCalledTimes(0);
jest.runOnlyPendingTimers();
expect(updateSpy).toBeCalledTimes(0);
});
test('should reinitialize a provider when save provider settings', async () => {
manager = new ExternalSecretsManager(settingsRepo, licenseMock, providersMock);
await manager.init();
const dummyInitSpy = jest.spyOn(DummyProvider.prototype, 'init');
await manager.setProviderSettings('dummy', {
test: 'value',
});
expect(dummyInitSpy).toBeCalledTimes(1);
});
});