diff --git a/packages/core/package.json b/packages/core/package.json index 00400791e..0325eb7be 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -62,6 +62,7 @@ "p-cancelable": "2.1.1", "pretty-bytes": "5.6.0", "qs": "6.11.0", + "ssh2": "1.15.0", "typedi": "0.10.0", "uuid": "8.3.2", "xml2js": "0.6.2" diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index e8ba90edd..0b2b164f1 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -101,6 +101,7 @@ import type { CallbackManager, INodeParameters, EnsureTypeOptions, + SSHTunnelFunctions, } from 'n8n-workflow'; import { ExpressionError, @@ -156,6 +157,7 @@ import Container from 'typedi'; import type { BinaryData } from './BinaryData/types'; import merge from 'lodash/merge'; import { InstanceSettings } from './InstanceSettings'; +import { SSHClientsManager } from './SSHClientsManager'; axios.defaults.timeout = 300000; // Prevent axios from adding x-form-www-urlencoded headers by default @@ -3276,6 +3278,11 @@ const getRequestHelperFunctions = ( }; }; +const getSSHTunnelFunctions = (): SSHTunnelFunctions => ({ + getSSHClient: async (credentials) => + await Container.get(SSHClientsManager).getClient(credentials), +}); + const getAllowedPaths = () => { const restrictFileAccessTo = process.env[RESTRICT_FILE_ACCESS_TO]; if (!restrictFileAccessTo) { @@ -3540,6 +3547,7 @@ export function getExecuteTriggerFunctions( }, helpers: { createDeferredPromise, + ...getSSHTunnelFunctions(), ...getRequestHelperFunctions(workflow, node, additionalData), ...getBinaryHelperFunctions(additionalData, workflow.id), returnJsonArray, @@ -3830,6 +3838,7 @@ export function getExecuteFunctions( createDeferredPromise, copyInputItems, ...getRequestHelperFunctions(workflow, node, additionalData), + ...getSSHTunnelFunctions(), ...getFileSystemHelperFunctions(node), ...getBinaryHelperFunctions(additionalData, workflow.id), assertBinaryData: (itemIndex, propertyName) => @@ -4010,6 +4019,7 @@ export function getExecuteSingleFunctions( export function getCredentialTestFunctions(): ICredentialTestFunctions { return { helpers: { + ...getSSHTunnelFunctions(), request: async (uriOrObject: string | object, options?: object) => { return await proxyRequestToAxios(undefined, undefined, undefined, uriOrObject, options); }, @@ -4088,7 +4098,10 @@ export function getLoadOptionsFunctions( options, ); }, - helpers: getRequestHelperFunctions(workflow, node, additionalData), + helpers: { + ...getSSHTunnelFunctions(), + ...getRequestHelperFunctions(workflow, node, additionalData), + }, }; })(workflow, node, path); } diff --git a/packages/core/src/SSHClientsManager.ts b/packages/core/src/SSHClientsManager.ts new file mode 100644 index 000000000..78126f96e --- /dev/null +++ b/packages/core/src/SSHClientsManager.ts @@ -0,0 +1,76 @@ +import { Service } from 'typedi'; +import { Client, type ConnectConfig } from 'ssh2'; +import { createHash } from 'node:crypto'; +import type { SSHCredentials } from 'n8n-workflow'; + +@Service() +export class SSHClientsManager { + readonly clients = new Map(); + + constructor() { + // Close all SSH connections when the process exits + process.on('exit', () => this.onShutdown()); + + if (process.env.NODE_ENV === 'test') return; + + // Regularly close stale SSH connections + setInterval(() => this.cleanupStaleConnections(), 60 * 1000); + } + + async getClient(credentials: SSHCredentials): Promise { + const { sshAuthenticateWith, sshHost, sshPort, sshUser } = credentials; + const sshConfig: ConnectConfig = { + host: sshHost, + port: sshPort, + username: sshUser, + ...(sshAuthenticateWith === 'password' + ? { password: credentials.sshPassword } + : { + privateKey: credentials.privateKey, + passphrase: credentials.passphrase ?? undefined, + }), + }; + + const clientHash = createHash('sha1').update(JSON.stringify(sshConfig)).digest('base64'); + + const existing = this.clients.get(clientHash); + if (existing) { + existing.lastUsed = new Date(); + return existing.client; + } + + return await new Promise((resolve, reject) => { + const sshClient = new Client(); + sshClient.once('error', reject); + sshClient.once('ready', () => { + sshClient.off('error', reject); + sshClient.once('close', () => this.clients.delete(clientHash)); + this.clients.set(clientHash, { + client: sshClient, + lastUsed: new Date(), + }); + resolve(sshClient); + }); + sshClient.connect(sshConfig); + }); + } + + onShutdown() { + for (const { client } of this.clients.values()) { + client.end(); + } + } + + cleanupStaleConnections() { + const { clients } = this; + if (clients.size === 0) return; + + const now = Date.now(); + for (const [hash, { client, lastUsed }] of clients.entries()) { + if (now - lastUsed.getTime() > 5 * 60 * 1000) { + client.end(); + clients.delete(hash); + } + } + } +} diff --git a/packages/core/test/SSHClientsManager.test.ts b/packages/core/test/SSHClientsManager.test.ts new file mode 100644 index 000000000..a7ceabe9f --- /dev/null +++ b/packages/core/test/SSHClientsManager.test.ts @@ -0,0 +1,67 @@ +import { Client } from 'ssh2'; +import type { SSHCredentials } from 'n8n-workflow'; +import { SSHClientsManager } from '@/SSHClientsManager'; + +describe('SSHClientsManager', () => { + const credentials: SSHCredentials = { + sshAuthenticateWith: 'password', + sshHost: 'example.com', + sshPort: 22, + sshUser: 'username', + sshPassword: 'password', + }; + + let sshClientsManager: SSHClientsManager; + const connectSpy = jest.spyOn(Client.prototype, 'connect'); + const endSpy = jest.spyOn(Client.prototype, 'end'); + + beforeEach(() => { + jest.clearAllMocks(); + jest.useFakeTimers(); + + sshClientsManager = new SSHClientsManager(); + connectSpy.mockImplementation(function (this: Client) { + this.emit('ready'); + return this; + }); + }); + + it('should create a new SSH client', async () => { + const client = await sshClientsManager.getClient(credentials); + + expect(client).toBeInstanceOf(Client); + }); + + it('should not create a new SSH client when connect fails', async () => { + connectSpy.mockImplementation(function (this: Client) { + throw new Error('Failed to connect'); + }); + await expect(sshClientsManager.getClient(credentials)).rejects.toThrow('Failed to connect'); + }); + + it('should reuse an existing SSH client', async () => { + const client1 = await sshClientsManager.getClient(credentials); + const client2 = await sshClientsManager.getClient(credentials); + + expect(client1).toBe(client2); + }); + + it('should close all SSH connections on process exit', async () => { + await sshClientsManager.getClient(credentials); + sshClientsManager.onShutdown(); + + expect(endSpy).toHaveBeenCalledTimes(1); + }); + + it('should cleanup stale SSH connections', async () => { + await sshClientsManager.getClient({ ...credentials, sshHost: 'host1' }); + await sshClientsManager.getClient({ ...credentials, sshHost: 'host2' }); + await sshClientsManager.getClient({ ...credentials, sshHost: 'host3' }); + + jest.advanceTimersByTime(6 * 60 * 1000); + sshClientsManager.cleanupStaleConnections(); + + expect(endSpy).toHaveBeenCalledTimes(3); + expect(sshClientsManager.clients.size).toBe(0); + }); +}); diff --git a/packages/nodes-base/credentials/MySql.credentials.ts b/packages/nodes-base/credentials/MySql.credentials.ts index f7efaf06e..73bff49be 100644 --- a/packages/nodes-base/credentials/MySql.credentials.ts +++ b/packages/nodes-base/credentials/MySql.credentials.ts @@ -1,4 +1,5 @@ import type { ICredentialType, INodeProperties } from 'n8n-workflow'; +import { sshTunnelProperties } from '@utils/sshTunnel.properties'; export class MySql implements ICredentialType { name = 'mySql'; @@ -97,120 +98,6 @@ export class MySql implements ICredentialType { type: 'string', default: '', }, - { - displayName: 'SSH Tunnel', - name: 'sshTunnel', - type: 'boolean', - default: false, - }, - { - displayName: 'SSH Authenticate with', - name: 'sshAuthenticateWith', - type: 'options', - default: 'password', - options: [ - { - name: 'Password', - value: 'password', - }, - { - name: 'Private Key', - value: 'privateKey', - }, - ], - displayOptions: { - show: { - sshTunnel: [true], - }, - }, - }, - { - displayName: 'SSH Host', - name: 'sshHost', - type: 'string', - default: 'localhost', - displayOptions: { - show: { - sshTunnel: [true], - }, - }, - }, - { - displayName: 'SSH Port', - name: 'sshPort', - type: 'number', - default: 22, - displayOptions: { - show: { - sshTunnel: [true], - }, - }, - }, - { - displayName: 'SSH MySQL Port', - name: 'sshMysqlPort', - type: 'number', - default: 3306, - displayOptions: { - show: { - sshTunnel: [true], - }, - }, - }, - { - displayName: 'SSH User', - name: 'sshUser', - type: 'string', - default: 'root', - displayOptions: { - show: { - sshTunnel: [true], - }, - }, - }, - { - displayName: 'SSH Password', - name: 'sshPassword', - type: 'string', - typeOptions: { - password: true, - }, - default: '', - displayOptions: { - show: { - sshTunnel: [true], - sshAuthenticateWith: ['password'], - }, - }, - }, - { - displayName: 'Private Key', - name: 'privateKey', - type: 'string', - typeOptions: { - rows: 4, - password: true, - }, - default: '', - displayOptions: { - show: { - sshTunnel: [true], - sshAuthenticateWith: ['privateKey'], - }, - }, - }, - { - displayName: 'Passphrase', - name: 'passphrase', - type: 'string', - default: '', - description: 'Passphase used to create the key, if no passphase was used leave empty', - displayOptions: { - show: { - sshTunnel: [true], - sshAuthenticateWith: ['privateKey'], - }, - }, - }, + ...sshTunnelProperties, ]; } diff --git a/packages/nodes-base/credentials/Postgres.credentials.ts b/packages/nodes-base/credentials/Postgres.credentials.ts index caf139e3d..d8916475b 100644 --- a/packages/nodes-base/credentials/Postgres.credentials.ts +++ b/packages/nodes-base/credentials/Postgres.credentials.ts @@ -1,4 +1,5 @@ import type { ICredentialType, INodeProperties } from 'n8n-workflow'; +import { sshTunnelProperties } from '@utils/sshTunnel.properties'; export class Postgres implements ICredentialType { name = 'postgres'; @@ -81,120 +82,6 @@ export class Postgres implements ICredentialType { type: 'number', default: 5432, }, - { - displayName: 'SSH Tunnel', - name: 'sshTunnel', - type: 'boolean', - default: false, - }, - { - displayName: 'SSH Authenticate with', - name: 'sshAuthenticateWith', - type: 'options', - default: 'password', - options: [ - { - name: 'Password', - value: 'password', - }, - { - name: 'Private Key', - value: 'privateKey', - }, - ], - displayOptions: { - show: { - sshTunnel: [true], - }, - }, - }, - { - displayName: 'SSH Host', - name: 'sshHost', - type: 'string', - default: 'localhost', - displayOptions: { - show: { - sshTunnel: [true], - }, - }, - }, - { - displayName: 'SSH Port', - name: 'sshPort', - type: 'number', - default: 22, - displayOptions: { - show: { - sshTunnel: [true], - }, - }, - }, - { - displayName: 'SSH Postgres Port', - name: 'sshPostgresPort', - type: 'number', - default: 5432, - displayOptions: { - show: { - sshTunnel: [true], - }, - }, - }, - { - displayName: 'SSH User', - name: 'sshUser', - type: 'string', - default: 'root', - displayOptions: { - show: { - sshTunnel: [true], - }, - }, - }, - { - displayName: 'SSH Password', - name: 'sshPassword', - type: 'string', - typeOptions: { - password: true, - }, - default: '', - displayOptions: { - show: { - sshTunnel: [true], - sshAuthenticateWith: ['password'], - }, - }, - }, - { - displayName: 'Private Key', - name: 'privateKey', - type: 'string', - typeOptions: { - rows: 4, - password: true, - }, - default: '', - displayOptions: { - show: { - sshTunnel: [true], - sshAuthenticateWith: ['privateKey'], - }, - }, - }, - { - displayName: 'Passphrase', - name: 'passphrase', - type: 'string', - default: '', - description: 'Passphase used to create the key, if no passphase was used leave empty', - displayOptions: { - show: { - sshTunnel: [true], - sshAuthenticateWith: ['privateKey'], - }, - }, - }, + ...sshTunnelProperties, ]; } diff --git a/packages/nodes-base/nodes/MySql/v2/actions/router.ts b/packages/nodes-base/nodes/MySql/v2/actions/router.ts index f5d75347c..0e9b49fa4 100644 --- a/packages/nodes-base/nodes/MySql/v2/actions/router.ts +++ b/packages/nodes-base/nodes/MySql/v2/actions/router.ts @@ -1,11 +1,8 @@ import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow'; import { NodeOperationError } from 'n8n-workflow'; -import { Client } from 'ssh2'; - -import type { QueryRunner } from '../helpers/interfaces'; - import { createPool } from '../transport'; +import type { MysqlNodeCredentials, QueryRunner } from '../helpers/interfaces'; import { configureQueryRunner } from '../helpers/utils'; import * as database from './database/Database.resource'; import type { MySqlType } from './node.type'; @@ -19,14 +16,9 @@ export async function router(this: IExecuteFunctions): Promise { - const credentials = credential.data as ICredentialDataDecryptedObject; + const credentials = credential.data as MysqlNodeCredentials; - let sshClient: Client | undefined = undefined; - - if (credentials.sshTunnel) { - sshClient = new Client(); - } - const pool = await createPool(credentials, {}, sshClient); + const pool = await createPool.call(this, credentials); try { const connection = await pool.getConnection(); @@ -30,9 +24,6 @@ export async function mysqlConnectionTest( message: error.message, }; } finally { - if (sshClient) { - sshClient.end(); - } await pool.end(); } diff --git a/packages/nodes-base/nodes/MySql/v2/methods/listSearch.ts b/packages/nodes-base/nodes/MySql/v2/methods/listSearch.ts index 2b6db1db0..b2bad4190 100644 --- a/packages/nodes-base/nodes/MySql/v2/methods/listSearch.ts +++ b/packages/nodes-base/nodes/MySql/v2/methods/listSearch.ts @@ -1,18 +1,13 @@ import type { IDataObject, ILoadOptionsFunctions, INodeListSearchResult } from 'n8n-workflow'; -import { Client } from 'ssh2'; import { createPool } from '../transport'; +import type { MysqlNodeCredentials } from '../helpers/interfaces'; export async function searchTables(this: ILoadOptionsFunctions): Promise { - const credentials = await this.getCredentials('mySql'); + const credentials = (await this.getCredentials('mySql')) as MysqlNodeCredentials; const nodeOptions = this.getNodeParameter('options', 0) as IDataObject; - let sshClient: Client | undefined = undefined; - - if (credentials.sshTunnel) { - sshClient = new Client(); - } - const pool = await createPool(credentials, nodeOptions, sshClient); + const pool = await createPool.call(this, credentials, nodeOptions); try { const connection = await pool.getConnection(); @@ -32,12 +27,7 @@ export async function searchTables(this: ILoadOptionsFunctions): Promise { - const credentials = await this.getCredentials('mySql'); + const credentials = (await this.getCredentials('mySql')) as MysqlNodeCredentials; const nodeOptions = this.getNodeParameter('options', 0) as IDataObject; - let sshClient: Client | undefined = undefined; - - if (credentials.sshTunnel) { - sshClient = new Client(); - } - const pool = await createPool(credentials, nodeOptions, sshClient); + const pool = await createPool.call(this, credentials, nodeOptions); try { const connection = await pool.getConnection(); @@ -39,12 +34,7 @@ export async function getColumns(this: ILoadOptionsFunctions): Promise { - if (credentials === undefined) { - throw new ApplicationError('Credentials not selected, select or add new credentials', { - level: 'warning', - }); - } - const { - ssl, - caCertificate, - clientCertificate, - clientPrivateKey, - sshTunnel, - sshHost, - sshUser, - sshPassword, - sshPort, - sshMysqlPort, - privateKey, - passphrase, - sshAuthenticateWith, - ...baseCredentials - } = credentials; - - if (ssl) { - baseCredentials.ssl = {}; - - if (caCertificate) { - baseCredentials.ssl.ca = formatPrivateKey(caCertificate as string); - } - - if (clientCertificate || clientPrivateKey) { - baseCredentials.ssl.cert = formatPrivateKey(clientCertificate as string); - baseCredentials.ssl.key = formatPrivateKey(clientPrivateKey as string); - } - } - const connectionOptions: mysql2.ConnectionOptions = { - ...baseCredentials, + host: credentials.host, + port: credentials.port, + database: credentials.database, + user: credentials.user, + password: credentials.password, multipleStatements: true, supportBigNumbers: true, }; + if (credentials.ssl) { + connectionOptions.ssl = {}; + + if (credentials.caCertificate) { + connectionOptions.ssl.ca = formatPrivateKey(credentials.caCertificate); + } + + if (credentials.clientCertificate || credentials.clientPrivateKey) { + connectionOptions.ssl.cert = formatPrivateKey(credentials.clientCertificate); + connectionOptions.ssl.key = formatPrivateKey(credentials.clientPrivateKey); + } + } + if (options?.nodeVersion && (options.nodeVersion as number) >= 2.1) { connectionOptions.dateStrings = true; } @@ -93,46 +55,39 @@ export async function createPool( connectionOptions.bigNumberStrings = true; } - if (!sshTunnel) { + if (!credentials.sshTunnel) { return mysql2.createPool(connectionOptions); } else { - if (!sshClient) { - throw new ApplicationError('SSH Tunnel is enabled but no SSH Client was provided', { - level: 'warning', - }); + if (credentials.sshAuthenticateWith === 'privateKey' && credentials.privateKey) { + credentials.privateKey = formatPrivateKey(credentials.privateKey as string); } + const sshClient = await this.helpers.getSSHClient(credentials); - const tunnelConfig = await createSshConnectConfig(credentials); - - const forwardConfig = { - srcHost: '127.0.0.1', - srcPort: sshMysqlPort as number, - dstHost: credentials.host as string, - dstPort: credentials.port as number, - }; - - const poolSetup = new Promise((resolve, reject) => { - sshClient - .on('ready', () => { - sshClient.forwardOut( - forwardConfig.srcHost, - forwardConfig.srcPort, - forwardConfig.dstHost, - forwardConfig.dstPort, - (err, stream) => { - if (err) reject(err); - const updatedDbServer = { - ...connectionOptions, - stream, - }; - const connection = mysql2.createPool(updatedDbServer); - resolve(connection); - }, - ); - }) - .connect(tunnelConfig); + // Find a free TCP port + const localPort = await new Promise((resolve) => { + const tempServer = createServer(); + tempServer.listen(0, LOCALHOST, () => { + resolve((tempServer.address() as AddressInfo).port); + tempServer.close(); + }); }); - return await poolSetup; + const stream = await new Promise((resolve, reject) => { + sshClient.forwardOut( + LOCALHOST, + localPort, + credentials.host, + credentials.port, + (err, clientChannel) => { + if (err) return reject(err); + resolve(clientChannel); + }, + ); + }); + + return mysql2.createPool({ + ...connectionOptions, + stream, + }); } } diff --git a/packages/nodes-base/nodes/Postgres/PostgresTrigger.functions.ts b/packages/nodes-base/nodes/Postgres/PostgresTrigger.functions.ts index e6830fc70..14b94b5ea 100644 --- a/packages/nodes-base/nodes/Postgres/PostgresTrigger.functions.ts +++ b/packages/nodes-base/nodes/Postgres/PostgresTrigger.functions.ts @@ -6,8 +6,9 @@ import type { INodeListSearchResult, INodeListSearchItems, } from 'n8n-workflow'; -import pgPromise from 'pg-promise'; -import type pg from 'pg-promise/typescript/pg-subset'; + +import type { PgpDatabase, PostgresNodeCredentials } from './v2/helpers/interfaces'; +import { configurePostgres } from './v2/transport'; export function prepareNames(id: string, mode: string, additionalFields: IDataObject) { let suffix = id.replace(/-/g, '_'); @@ -35,7 +36,7 @@ export function prepareNames(id: string, mode: string, additionalFields: IDataOb export async function pgTriggerFunction( this: ITriggerFunctions, - db: pgPromise.IDatabase<{}, pg.IClient>, + db: PgpDatabase, additionalFields: IDataObject, functionName: string, triggerName: string, @@ -86,43 +87,12 @@ export async function pgTriggerFunction( } export async function initDB(this: ITriggerFunctions | ILoadOptionsFunctions) { - const credentials = await this.getCredentials('postgres'); + const credentials = (await this.getCredentials('postgres')) as PostgresNodeCredentials; const options = this.getNodeParameter('options', {}) as { connectionTimeout?: number; delayClosingIdleConnection?: number; }; - const pgp = pgPromise({ - // prevent spam in console "WARNING: Creating a duplicate database object for the same connection." - noWarnings: true, - }); - const config: IDataObject = { - host: credentials.host as string, - port: credentials.port as number, - database: credentials.database as string, - user: credentials.user as string, - password: credentials.password as string, - keepAlive: true, - }; - - if (options.connectionTimeout) { - config.connectionTimeoutMillis = options.connectionTimeout * 1000; - } - - if (options.delayClosingIdleConnection) { - config.keepAliveInitialDelayMillis = options.delayClosingIdleConnection * 1000; - } - - if (credentials.allowUnauthorizedCerts === true) { - config.ssl = { - rejectUnauthorized: false, - }; - } else { - config.ssl = !['disable', undefined].includes(credentials.ssl as string | undefined); - config.sslmode = (credentials.ssl as string) || 'disable'; - } - - const db = pgp(config); - return { db, pgp }; + return await configurePostgres.call(this, credentials, options); } export async function searchSchema(this: ILoadOptionsFunctions): Promise { diff --git a/packages/nodes-base/nodes/Postgres/v2/actions/router.ts b/packages/nodes-base/nodes/Postgres/v2/actions/router.ts index ec9d97499..ccdcad70a 100644 --- a/packages/nodes-base/nodes/Postgres/v2/actions/router.ts +++ b/packages/nodes-base/nodes/Postgres/v2/actions/router.ts @@ -21,7 +21,7 @@ export async function router(this: IExecuteFunctions): Promise; export type PgpDatabase = pgPromise.IDatabase<{}, pg.IClient>; export type PgpConnectionParameters = pg.IConnectionParameters; -export type ConnectionsData = { db: PgpDatabase; pgp: PgpClient; sshClient?: Client }; +export type ConnectionsData = { db: PgpDatabase; pgp: PgpClient }; export type QueriesRunner = ( queries: QueryWithValues[], @@ -51,7 +50,6 @@ export type PostgresNodeOptions = { }; export type PostgresNodeCredentials = { - sshAuthenticateWith: 'password' | 'privateKey'; host: string; port: number; database: string; @@ -59,12 +57,9 @@ export type PostgresNodeCredentials = { password: string; allowUnauthorizedCerts?: boolean; ssl?: 'disable' | 'allow' | 'require' | 'verify' | 'verify-full'; - sshTunnel?: boolean; - sshHost?: string; - sshPort?: number; - sshPostgresPort?: number; - sshUser?: string; - sshPassword?: string; - privateKey?: string; - passphrase?: string; -}; +} & ( + | { sshTunnel: false } + | ({ + sshTunnel: true; + } & SSHCredentials) +); diff --git a/packages/nodes-base/nodes/Postgres/v2/methods/credentialTest.ts b/packages/nodes-base/nodes/Postgres/v2/methods/credentialTest.ts index 1b4d588fa..59cb83b5d 100644 --- a/packages/nodes-base/nodes/Postgres/v2/methods/credentialTest.ts +++ b/packages/nodes-base/nodes/Postgres/v2/methods/credentialTest.ts @@ -4,7 +4,6 @@ import type { INodeCredentialTestResult, } from 'n8n-workflow'; -import { Client } from 'ssh2'; import { configurePostgres } from '../transport'; import type { PgpClient, PostgresNodeCredentials } from '../helpers/interfaces'; @@ -15,13 +14,11 @@ export async function postgresConnectionTest( ): Promise { const credentials = credential.data as PostgresNodeCredentials; - let sshClientCreated: Client | undefined = new Client(); let pgpClientCreated: PgpClient | undefined; try { - const { db, pgp, sshClient } = await configurePostgres(credentials, {}, sshClientCreated); + const { db, pgp } = await configurePostgres.call(this, credentials, {}); - sshClientCreated = sshClient; pgpClientCreated = pgp; await db.connect(); @@ -45,9 +42,6 @@ export async function postgresConnectionTest( message, }; } finally { - if (sshClientCreated) { - sshClientCreated.end(); - } if (pgpClientCreated) { pgpClientCreated.end(); } diff --git a/packages/nodes-base/nodes/Postgres/v2/methods/listSearch.ts b/packages/nodes-base/nodes/Postgres/v2/methods/listSearch.ts index 68c5c3d9e..f8612f4df 100644 --- a/packages/nodes-base/nodes/Postgres/v2/methods/listSearch.ts +++ b/packages/nodes-base/nodes/Postgres/v2/methods/listSearch.ts @@ -7,7 +7,7 @@ export async function schemaSearch(this: ILoadOptionsFunctions): Promise { const credentials = (await this.getCredentials('postgres')) as PostgresNodeCredentials; - const { db, sshClient } = await configurePostgres(credentials); + const { db } = await configurePostgres.call(this, credentials); const schema = this.getNodeParameter('schema', 0, { extractValue: true, @@ -89,12 +89,7 @@ export async function getMappingColumns( }), ); return { fields }; - } catch (error) { - throw error; } finally { - if (sshClient) { - sshClient.end(); - } if (!db.$pool.ending) await db.$pool.end(); } } diff --git a/packages/nodes-base/nodes/Postgres/v2/transport/index.ts b/packages/nodes-base/nodes/Postgres/v2/transport/index.ts index 773680581..14c98a2bf 100644 --- a/packages/nodes-base/nodes/Postgres/v2/transport/index.ts +++ b/packages/nodes-base/nodes/Postgres/v2/transport/index.ts @@ -1,72 +1,26 @@ -import type { Server } from 'net'; -import { createServer } from 'net'; -import { Client } from 'ssh2'; -import type { ConnectConfig } from 'ssh2'; - -import type { IDataObject } from 'n8n-workflow'; - +import { createServer, type AddressInfo } from 'node:net'; import pgPromise from 'pg-promise'; import type { - PgpDatabase, + IExecuteFunctions, + ICredentialTestFunctions, + ILoadOptionsFunctions, + ITriggerFunctions, +} from 'n8n-workflow'; + +import { formatPrivateKey } from '@utils/utilities'; +import type { + ConnectionsData, + PgpConnectionParameters, PostgresNodeCredentials, PostgresNodeOptions, } from '../helpers/interfaces'; -import { formatPrivateKey } from '@utils/utilities'; +import { LOCALHOST } from '@utils/constants'; -async function createSshConnectConfig(credentials: PostgresNodeCredentials) { - if (credentials.sshAuthenticateWith === 'password') { - return { - host: credentials.sshHost as string, - port: credentials.sshPort as number, - username: credentials.sshUser as string, - password: credentials.sshPassword as string, - } as ConnectConfig; - } else { - const options: ConnectConfig = { - host: credentials.sshHost as string, - username: credentials.sshUser as string, - port: credentials.sshPort as number, - privateKey: formatPrivateKey(credentials.privateKey as string), - }; - - if (credentials.passphrase) { - options.passphrase = credentials.passphrase; - } - - return options; - } -} - -export async function configurePostgres( +const getPostgresConfig = ( credentials: PostgresNodeCredentials, options: PostgresNodeOptions = {}, - createdSshClient?: Client, -) { - const pgp = pgPromise({ - // prevent spam in console "WARNING: Creating a duplicate database object for the same connection." - // duplicate connections created when auto loading parameters, they are closed imidiatly after, but several could be open at the same time - noWarnings: true, - }); - - if (typeof options.nodeVersion === 'number' && options.nodeVersion >= 2.1) { - // Always return dates as ISO strings - [pgp.pg.types.builtins.TIMESTAMP, pgp.pg.types.builtins.TIMESTAMPTZ].forEach((type) => { - pgp.pg.types.setTypeParser(type, (value: string) => { - return new Date(value).toISOString(); - }); - }); - } - - if (options.largeNumbersOutput === 'numbers') { - pgp.pg.types.setTypeParser(20, (value: string) => { - return parseInt(value, 10); - }); - pgp.pg.types.setTypeParser(1700, (value: string) => { - return parseFloat(value); - }); - } - - const dbConfig: IDataObject = { +) => { + const dbConfig: PgpConnectionParameters = { host: credentials.host, port: credentials.port, database: credentials.database, @@ -89,70 +43,91 @@ export async function configurePostgres( }; } else { dbConfig.ssl = !['disable', undefined].includes(credentials.ssl as string | undefined); + // @ts-ignore these typings need to be updated dbConfig.sslmode = credentials.ssl || 'disable'; } + return dbConfig; +}; + +export async function configurePostgres( + this: IExecuteFunctions | ICredentialTestFunctions | ILoadOptionsFunctions | ITriggerFunctions, + credentials: PostgresNodeCredentials, + options: PostgresNodeOptions = {}, +): Promise { + const pgp = pgPromise({ + // prevent spam in console "WARNING: Creating a duplicate database object for the same connection." + // duplicate connections created when auto loading parameters, they are closed immediately after, but several could be open at the same time + noWarnings: true, + }); + + if (typeof options.nodeVersion === 'number' && options.nodeVersion >= 2.1) { + // Always return dates as ISO strings + [pgp.pg.types.builtins.TIMESTAMP, pgp.pg.types.builtins.TIMESTAMPTZ].forEach((type) => { + pgp.pg.types.setTypeParser(type, (value: string) => { + return new Date(value).toISOString(); + }); + }); + } + + if (options.largeNumbersOutput === 'numbers') { + pgp.pg.types.setTypeParser(20, (value: string) => { + return parseInt(value, 10); + }); + pgp.pg.types.setTypeParser(1700, (value: string) => { + return parseFloat(value); + }); + } + + const dbConfig = getPostgresConfig(credentials, options); + if (!credentials.sshTunnel) { const db = pgp(dbConfig); return { db, pgp }; } else { - const sshClient = createdSshClient || new Client(); + if (credentials.sshAuthenticateWith === 'privateKey' && credentials.privateKey) { + credentials.privateKey = formatPrivateKey(credentials.privateKey); + } + const sshClient = await this.helpers.getSSHClient(credentials); - const tunnelConfig = await createSshConnectConfig(credentials); + // Create a TCP proxy listening on a random available port + const proxy = createServer(); + const proxyPort = await new Promise((resolve) => { + proxy.listen(0, LOCALHOST, () => { + resolve((proxy.address() as AddressInfo).port); + }); + }); - const localHost = '127.0.0.1'; - const localPort = credentials.sshPostgresPort as number; - - let proxy: Server | undefined; - - const db = await new Promise((resolve, reject) => { - let sshClientReady = false; - - proxy = createServer((socket) => { - if (!sshClientReady) return socket.destroy(); + const close = () => { + proxy.close(); + sshClient.off('end', close); + sshClient.off('error', close); + }; + sshClient.on('end', close); + sshClient.on('error', close); + await new Promise((resolve, reject) => { + proxy.on('error', (err) => reject(err)); + proxy.on('connection', (localSocket) => { sshClient.forwardOut( - socket.remoteAddress as string, - socket.remotePort as number, + LOCALHOST, + localSocket.remotePort!, credentials.host, credentials.port, - (err, stream) => { - if (err) reject(err); - - socket.pipe(stream); - stream.pipe(socket); + (err, clientChannel) => { + if (err) { + proxy.close(); + localSocket.destroy(); + } else { + localSocket.pipe(clientChannel); + clientChannel.pipe(localSocket); + } }, ); - }).listen(localPort, localHost); - - proxy.on('error', (err) => { - reject(err); - }); - - sshClient.connect(tunnelConfig); - - sshClient.on('ready', () => { - sshClientReady = true; - - const updatedDbConfig = { - ...dbConfig, - port: localPort, - host: localHost, - }; - const dbConnection = pgp(updatedDbConfig); - resolve(dbConnection); - }); - - sshClient.on('error', (err) => { - reject(err); - }); - - sshClient.on('end', async () => { - if (proxy) proxy.close(); }); + resolve(); }).catch((err) => { - if (proxy) proxy.close(); - if (sshClient) sshClient.end(); + proxy.close(); let message = err.message; let description = err.description; @@ -183,6 +158,11 @@ export async function configurePostgres( throw err; }); - return { db, pgp, sshClient }; + const db = pgp({ + ...dbConfig, + port: proxyPort, + host: LOCALHOST, + }); + return { db, pgp }; } } diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index 159c6720d..c884a1ea6 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -826,7 +826,6 @@ "@types/mailparser": "^3.4.4", "@types/mime-types": "^2.1.0", "@types/mssql": "^9.1.5", - "@types/node-ssh": "^7.0.1", "@types/nodemailer": "^6.4.14", "@types/promise-ftp": "^1.3.4", "@types/rfc2047": "^2.0.1", @@ -880,7 +879,7 @@ "n8n-workflow": "workspace:*", "nanoid": "3.3.6", "node-html-markdown": "1.2.0", - "node-ssh": "12.0.5", + "node-ssh": "13.2.0", "nodemailer": "6.9.9", "otpauth": "9.1.1", "pdfjs-dist": "2.16.105", diff --git a/packages/nodes-base/utils/constants.ts b/packages/nodes-base/utils/constants.ts index 095d71f1e..400bbc586 100644 --- a/packages/nodes-base/utils/constants.ts +++ b/packages/nodes-base/utils/constants.ts @@ -1,2 +1,4 @@ export const NODE_RAN_MULTIPLE_TIMES_WARNING = "This node ran multiple times - once for each input item. You can change this by setting 'execute once' in the node settings. More Info"; + +export const LOCALHOST = '127.0.0.1'; diff --git a/packages/nodes-base/utils/sshTunnel.properties.ts b/packages/nodes-base/utils/sshTunnel.properties.ts new file mode 100644 index 000000000..ec72f2ed0 --- /dev/null +++ b/packages/nodes-base/utils/sshTunnel.properties.ts @@ -0,0 +1,108 @@ +import type { INodeProperties } from 'n8n-workflow'; + +export const sshTunnelProperties: INodeProperties[] = [ + { + displayName: 'SSH Tunnel', + name: 'sshTunnel', + type: 'boolean', + default: false, + }, + { + displayName: 'SSH Authenticate with', + name: 'sshAuthenticateWith', + type: 'options', + default: 'password', + options: [ + { + name: 'Password', + value: 'password', + }, + { + name: 'Private Key', + value: 'privateKey', + }, + ], + displayOptions: { + show: { + sshTunnel: [true], + }, + }, + }, + { + displayName: 'SSH Host', + name: 'sshHost', + type: 'string', + default: 'localhost', + displayOptions: { + show: { + sshTunnel: [true], + }, + }, + }, + { + displayName: 'SSH Port', + name: 'sshPort', + type: 'number', + default: 22, + displayOptions: { + show: { + sshTunnel: [true], + }, + }, + }, + { + displayName: 'SSH User', + name: 'sshUser', + type: 'string', + default: 'root', + displayOptions: { + show: { + sshTunnel: [true], + }, + }, + }, + { + displayName: 'SSH Password', + name: 'sshPassword', + type: 'string', + typeOptions: { + password: true, + }, + default: '', + displayOptions: { + show: { + sshTunnel: [true], + sshAuthenticateWith: ['password'], + }, + }, + }, + { + displayName: 'Private Key', + name: 'sshPrivateKey', + type: 'string', + typeOptions: { + rows: 4, + password: true, + }, + default: '', + displayOptions: { + show: { + sshTunnel: [true], + sshAuthenticateWith: ['privateKey'], + }, + }, + }, + { + displayName: 'Passphrase', + name: 'sshPassphrase', + type: 'string', + default: '', + description: 'Passphrase used to create the key, if no passphrase was used leave empty', + displayOptions: { + show: { + sshTunnel: [true], + sshAuthenticateWith: ['privateKey'], + }, + }, + }, +]; diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index 0830935be..68d7e6993 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -8,6 +8,7 @@ import type { SecureContextOptions } from 'tls'; import type { Readable } from 'stream'; import type { URLSearchParams } from 'url'; import type { RequestBodyMatcher } from 'nock'; +import type { Client as SSHClient } from 'ssh2'; import type { AuthenticationMethod } from './Authentication'; import type { CODE_EXECUTION_MODES, CODE_LANGUAGES, LOG_LEVELS } from './Constants'; @@ -717,7 +718,7 @@ export type ICredentialTestFunction = ( ) => Promise; export interface ICredentialTestFunctions { - helpers: { + helpers: SSHTunnelFunctions & { request: (uriOrObject: string | object, options?: object) => Promise; }; } @@ -816,6 +817,28 @@ export interface RequestHelperFunctions { ): Promise; } +export type SSHCredentials = { + sshHost: string; + sshPort: number; + sshUser: string; +} & ( + | { + sshAuthenticateWith: 'password'; + sshPassword: string; + } + | { + sshAuthenticateWith: 'privateKey'; + // TODO: rename this to `sshPrivateKey` + privateKey: string; + // TODO: rename this to `sshPassphrase` + passphrase?: string; + } +); + +export interface SSHTunnelFunctions { + getSSHClient(credentials: SSHCredentials): Promise; +} + export type NodeTypeAndVersion = { name: string; type: string; @@ -899,6 +922,7 @@ export type IExecuteFunctions = ExecuteFunctions.GetNodeParameterFn & BaseHelperFunctions & BinaryHelperFunctions & FileSystemHelperFunctions & + SSHTunnelFunctions & JsonHelperFunctions & { normalizeItems(items: INodeExecutionData | INodeExecutionData[]): INodeExecutionData[]; constructExecutionMetaData( @@ -948,7 +972,7 @@ export interface ILoadOptionsFunctions extends FunctionsBase { options?: IGetNodeParameterOptions, ): NodeParameterValueType | object | undefined; getCurrentNodeParameters(): INodeParameters | undefined; - helpers: RequestHelperFunctions; + helpers: RequestHelperFunctions & SSHTunnelFunctions; } export interface IPollFunctions @@ -986,6 +1010,7 @@ export interface ITriggerFunctions helpers: RequestHelperFunctions & BaseHelperFunctions & BinaryHelperFunctions & + SSHTunnelFunctions & JsonHelperFunctions; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e4fd7576c..b6b3a5699 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -929,6 +929,9 @@ importers: qs: specifier: 6.11.0 version: 6.11.0 + ssh2: + specifier: 1.15.0 + version: 1.15.0 typedi: specifier: 0.10.0 version: 0.10.0(patch_hash=sk6omkefrosihg7lmqbzh7vfxe) @@ -1456,8 +1459,8 @@ importers: specifier: 1.2.0 version: 1.2.0 node-ssh: - specifier: 12.0.5 - version: 12.0.5 + specifier: 13.2.0 + version: 13.2.0 nodemailer: specifier: 6.9.9 version: 6.9.9 @@ -1579,9 +1582,6 @@ importers: '@types/mssql': specifier: ^9.1.5 version: 9.1.5 - '@types/node-ssh': - specifier: ^7.0.1 - version: 7.0.1 '@types/nodemailer': specifier: ^6.4.14 version: 6.4.14 @@ -5629,9 +5629,6 @@ packages: '@types/node-fetch@2.6.4': resolution: {integrity: sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==} - '@types/node-ssh@7.0.1': - resolution: {integrity: sha512-98EuH7UQl/WWwwDxpbANQ76HwBdzcSnC9zLSdrtVW7jjYeOTQ6TxBygbGwzZR4ho1agbd941UnHCdrXz2sS8JQ==} - '@types/node@18.16.16': resolution: {integrity: sha512-NpaM49IGQQAUlBhHMF82QH80J08os4ZmyF9MkpCzWAGuOHqE4gTEbhzd7L3l5LmWuZ6E0OiC1FweQ4tsiW35+g==} @@ -5725,9 +5722,6 @@ packages: '@types/ssh2-sftp-client@5.3.2': resolution: {integrity: sha512-s5R3hsnI3/7Ar57LG++gm2kxgONHtOZY2A3AgGzEwiJlHR8j7MRPDw1n/hG6oMnOUJ4zuoLNtDXgDfmmxV4lDA==} - '@types/ssh2-streams@0.1.9': - resolution: {integrity: sha512-I2J9jKqfmvXLR5GomDiCoHrEJ58hAOmFrekfFqmCFd+A6gaEStvWnPykoWUwld1PNg4G5ag1LwdA+Lz1doRJqg==} - '@types/ssh2@1.11.6': resolution: {integrity: sha512-8Mf6bhzYYBLEB/G6COux7DS/F5bCWwojv/qFo2yH/e4cLzAavJnxvFXrYW59iKfXdhG6OmzJcXDasgOb/s0rxw==} @@ -6707,8 +6701,8 @@ packages: buffer@6.0.3: resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} - buildcheck@0.0.3: - resolution: {integrity: sha512-pziaA+p/wdVImfcbsZLNF32EiWyujlQLwolMqUQE8xpKNOH7KmZQaY8sXN7DGOEzPAElo9QTaeNRfGnf3iOJbA==} + buildcheck@0.0.6: + resolution: {integrity: sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==} engines: {node: '>=10.0.0'} builtin-modules@3.3.0: @@ -7178,8 +7172,8 @@ packages: core-util-is@1.0.3: resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} - cpu-features@0.0.4: - resolution: {integrity: sha512-fKiZ/zp1mUwQbnzb9IghXtHtDoTMtNeb8oYGx6kX2SYfhnG0HNdBEBIzB9b5KlXu5DQPhfy3mInbBxFcgwAr3A==} + cpu-features@0.0.10: + resolution: {integrity: sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==} engines: {node: '>=10.0.0'} crelt@1.0.5: @@ -10649,8 +10643,8 @@ packages: resolution: {integrity: sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==} engines: {node: '>=12.0.0'} - nan@2.17.0: - resolution: {integrity: sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==} + nan@2.20.0: + resolution: {integrity: sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==} nanoclone@0.2.1: resolution: {integrity: sha512-wynEP02LmIbLpcYw8uBKpcfF6dmg2vcpKqxeH5UcoKEYdExslsdUA4ugFauuaeYdTB76ez6gJW8XAZ6CgkXYxA==} @@ -10773,8 +10767,8 @@ packages: node-rsa@1.1.1: resolution: {integrity: sha512-Jd4cvbJMryN21r5HgxQOpMEqv+ooke/korixNNK3mGqfGJmy0M77WDDzo/05969+OkMy3XW1UuZsSmW9KQm7Fw==} - node-ssh@12.0.5: - resolution: {integrity: sha512-uN2GTGdBRUUKkZmcNBr9OM+xKL6zq74emnkSyb1TshBdVWegj3boue6QallQeqZzo7YGVheP5gAovUL+8hZSig==} + node-ssh@13.2.0: + resolution: {integrity: sha512-7vsKR2Bbs66th6IWCy/7SN4MSwlVt+G6QrHB631BjRUM8/LmvDugtYhi0uAmgvHS/+PVurfNBOmELf30rm0MZg==} engines: {node: '>= 10'} nodemailer@6.9.9: @@ -12491,8 +12485,8 @@ packages: resolution: {integrity: sha512-Bmq4Uewu3e0XOwu5bnPbiS5KRQYv+dff5H6+85V4GZrPrt0Fkt1nUH+uXanyAkoNxUpzjnAPEEoLdOaBO9c3xw==} engines: {node: '>=10.24.1'} - ssh2@1.11.0: - resolution: {integrity: sha512-nfg0wZWGSsfUe/IBJkXVll3PEZ//YH2guww+mP88gTpuSU4FtZN7zu9JoeTGOyCNx2dTDtT9fOpWwlzyj4uOOw==} + ssh2@1.15.0: + resolution: {integrity: sha512-C0PHgX4h6lBxYx7hcXwu3QWdh4tg6tZZsTfXcdvc5caW/EMxaB4H9dWsl7qk+F7LAW762hp8VbXOX7x4xUYvEw==} engines: {node: '>=10.16.0'} sshpk@1.17.0: @@ -19715,12 +19709,6 @@ snapshots: '@types/node': 18.16.16 form-data: 3.0.1 - '@types/node-ssh@7.0.1': - dependencies: - '@types/node': 18.16.16 - '@types/ssh2': 1.11.6 - '@types/ssh2-streams': 0.1.9 - '@types/node@18.16.16': {} '@types/nodemailer@6.4.14': @@ -19818,10 +19806,6 @@ snapshots: dependencies: '@types/ssh2': 1.11.6 - '@types/ssh2-streams@0.1.9': - dependencies: - '@types/node': 18.16.16 - '@types/ssh2@1.11.6': dependencies: '@types/node': 18.16.16 @@ -21037,7 +21021,7 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 - buildcheck@0.0.3: + buildcheck@0.0.6: optional: true builtin-modules@3.3.0: {} @@ -21566,10 +21550,10 @@ snapshots: core-util-is@1.0.3: {} - cpu-features@0.0.4: + cpu-features@0.0.10: dependencies: - buildcheck: 0.0.3 - nan: 2.17.0 + buildcheck: 0.0.6 + nan: 2.20.0 optional: true crelt@1.0.5: {} @@ -25671,7 +25655,7 @@ snapshots: dependencies: lru-cache: 7.18.3 - nan@2.17.0: + nan@2.20.0: optional: true nanoclone@0.2.1: {} @@ -25785,14 +25769,14 @@ snapshots: dependencies: asn1: 0.2.6 - node-ssh@12.0.5: + node-ssh@13.2.0: dependencies: is-stream: 2.0.1 make-dir: 3.1.0 sb-promise-queue: 2.1.0 sb-scandir: 3.1.0 shell-escape: 0.2.0 - ssh2: 1.11.0 + ssh2: 1.15.0 nodemailer@6.9.9: {} @@ -27775,15 +27759,15 @@ snapshots: dependencies: concat-stream: 2.0.0 promise-retry: 2.0.1 - ssh2: 1.11.0 + ssh2: 1.15.0 - ssh2@1.11.0: + ssh2@1.15.0: dependencies: asn1: 0.2.6 bcrypt-pbkdf: 1.0.2 optionalDependencies: - cpu-features: 0.0.4 - nan: 2.17.0 + cpu-features: 0.0.10 + nan: 2.20.0 sshpk@1.17.0: dependencies: