mohiit1502/Automata
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Environments release using source control (#6653)

Browse Source 
* initial telemetry setup and adjusted pull return

* quicksave before merge

* feat: add conflicting workflow list to pull modal

* feat: update source control pull modal

* fix: fix linting issue

* feat: add Enter keydown event for submitting source control push modal (no-changelog)

feat: add Enter keydown event for submitting source control push modal

* quicksave

* user workflow table for export

* improve telemetry data

* pull api telemetry

* fix lint

* Copy tweaks.

* remove authorName and authorEmail and pick from user

* rename owners.json to workflow_owners.json

* ignore credential conflicts on pull

* feat: several push/pull flow changes and design update

* pull and push return same data format

* fix: add One last step toast for successful pull

* feat: add up to date pull toast

* fix: add proper Learn more link for push and pull modals

* do not await tracking being sent

* fix import

* fix await

* add more sourcecontrolfile status

* Minor copy tweak for "More info".

* Minor copy tweak for "More info".

* ignore variable_stub conflicts on pull

* ignore whitespace differences

* do not show remote workflows that are not yet created

* fix telemetry

* fix toast when pulling deleted wf

* lint fix

* refactor and make some imports dynamic

* fix variable edit validation

* fix telemetry response

* improve telemetry

* fix unintenional delete commit

* fix status unknown issue

* fix up to date toast

* do not export active state and reapply versionid

* use update instead of upsert

* fix: show all workflows when clicking push to git

* feat: update Up to date pull translation

* fix: update read only env checks

* do not update versionid of only active flag changes

* feat: prevent access to new workflow and templates import when read only env

* feat: send only active state and version if workflow state is not dirty

* fix: Detect when only active state has changed and prevent generation a new version ID

* feat: improve readonly env messages

* make getPreferences public

* fix telemetry issue

* fix: add partial workflow update based on dirty state when changing active state

* update unit tests

* fix: remove unsaved changes check in readOnlyEnv

* fix: disable push to git button when read onyl env

* fix: update readonly toast duration

* fix: fix pinning and title input in protected mode

* initial commit (NOT working)

* working push

* cleanup and implement pull

* fix getstatus

* update import to new method

* var and tag diffs are no conflicts

* only show pull conflict for workflows

* refactor and ignore faulty credentials

* add sanitycheck for missing git folder

* prefer fetch over pull and limit depth to 1

* back to pull...

* fix setting branch on initial connect

* fix test

* remove clean workfolder

* refactor: Remove some unnecessary code

* Fixed links to docs.

* fix getstatus query params

* lint fix

* dialog to show local and remote name on conflict

* only show remote name on conflict

* fix credential expression export

* fix: Broken test

* dont show toast on pull with empty var/tags and refactor

* apply frontend changes from old branch

* fix tag with same name import

* fix buttons shown for non instance owners

* prepare local storage key for removal

* refactor: Change wording on pushing and pulling

* refactor: Change menu item

* test: Fix broken test

* Update packages/cli/src/environments/sourceControl/types/sourceControlPushWorkFolder.ts

Co-authored-by: Iván Ovejero <ivov.src@gmail.com>

---------

Co-authored-by: Alex Grozav <alex@grozav.com>
Co-authored-by: Giulio Andreini <g.andreini@gmail.com>
Co-authored-by: Omar Ajoue <krynble@gmail.com>
Co-authored-by: Iván Ovejero <ivov.src@gmail.com>
This commit is contained in:
Michael Auerswald
2023-07-26 09:25:01 +02:00
committed by GitHub
parent bcfc5e717b
commit fc7aa8bd66
51 changed files with 2210 additions and 1064 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
packages/cli/src/InternalHooks.ts
View File

@@ -54,6 +54,14 @@ function userToPayload(user: User): {
export class InternalHooks implements IInternalHooksClass {
private instanceId: string;
public get telemetryInstanceId(): string {
return this.instanceId;
}
public get telemetryInstance(): Telemetry {
return this.telemetry;
}
constructor(
private telemetry: Telemetry,
private nodeTypes: NodeTypes,
@@ -1043,4 +1051,53 @@ export class InternalHooks implements IInternalHooksClass {
async onVariableCreated(createData: { variable_type: string }): Promise<void> {
return this.telemetry.track('User created variable', createData);
}
async onSourceControlSettingsUpdated(data: {
branch_name: string;
read_only_instance: boolean;
repo_type: 'github' | 'gitlab' | 'other';
connected: boolean;
}): Promise<void> {
return this.telemetry.track('User updated source control settings', data);
}
async onSourceControlUserStartedPullUI(data: {
workflow_updates: number;
workflow_conflicts: number;
cred_conflicts: number;
}): Promise<void> {
return this.telemetry.track('User started pull via UI', data);
}
async onSourceControlUserFinishedPullUI(data: { workflow_updates: number }): Promise<void> {
return this.telemetry.track('User finished pull via UI', {
workflow_updates: data.workflow_updates,
});
}
async onSourceControlUserPulledAPI(data: {
workflow_updates: number;
forced: boolean;
}): Promise<void> {
return this.telemetry.track('User pulled via API', data);
}
async onSourceControlUserStartedPushUI(data: {
workflows_eligible: number;
workflows_eligible_with_conflicts: number;
creds_eligible: number;
creds_eligible_with_conflicts: number;
variables_eligible: number;
}): Promise<void> {
return this.telemetry.track('User started push via UI', data);
}
async onSourceControlUserFinishedPushUI(data: {
workflows_eligible: number;
workflows_pushed: number;
creds_pushed: number;
variables_pushed: number;
}): Promise<void> {
return this.telemetry.track('User finished push via UI', data);
}
}

2
packages/cli/src/Ldap/helpers.ts
View File

@@ -217,7 +217,7 @@ export const handleLdapInit = async (): Promise<void> => {
try {
await setGlobalLdapConfigVariables(ldapConfig);
} catch (error) {
Logger.error(
Logger.warn(
`Cannot set LDAP login enabled state when an authentication method other than email or ldap is active (current: ${getCurrentAuthenticationMethod()})`,
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
error,

18
packages/cli/src/PublicApi/v1/handlers/sourceControl/sourceControl.handler.ts
View File

@@ -6,7 +6,11 @@ import { authorize } from '../../shared/middlewares/global.middleware';
import type { ImportResult } from '@/environments/sourceControl/types/importResult';
import { SourceControlService } from '@/environments/sourceControl/sourceControl.service.ee';
import { SourceControlPreferencesService } from '@/environments/sourceControl/sourceControlPreferences.service.ee';
import { isSourceControlLicensed } from '@/environments/sourceControl/sourceControlHelper.ee';
import {
getTrackingInformationFromPullResult,
isSourceControlLicensed,
} from '@/environments/sourceControl/sourceControlHelper.ee';
import { InternalHooks } from '@/InternalHooks';
export = {
pull: [
@@ -32,12 +36,16 @@ export = {
force: req.body.force,
variables: req.body.variables,
userId: req.user.id,
importAfterPull: true,
});
if ((result as ImportResult)?.workflows) {
return res.status(200).send(result as ImportResult);
if (result.statusCode === 200) {
void Container.get(InternalHooks).onSourceControlUserPulledAPI({
...getTrackingInformationFromPullResult(result.statusResult),
forced: req.body.force ?? false,
});
return res.status(200).send(result.statusResult);
} else {
return res.status(409).send(result);
return res.status(409).send(result.statusResult);
}
} catch (error) {
return res.status(400).send((error as { message: string }).message);

4
packages/cli/src/environments/sourceControl/constants.ts
View File

@@ -5,7 +5,7 @@ export const SOURCE_CONTROL_WORKFLOW_EXPORT_FOLDER = 'workflows';
export const SOURCE_CONTROL_CREDENTIAL_EXPORT_FOLDER = 'credential_stubs';
export const SOURCE_CONTROL_VARIABLES_EXPORT_FILE = 'variable_stubs.json';
export const SOURCE_CONTROL_TAGS_EXPORT_FILE = 'tags.json';
export const SOURCE_CONTROL_OWNERS_EXPORT_FILE = 'owners.json';
export const SOURCE_CONTROL_OWNERS_EXPORT_FILE = 'workflow_owners.json';
export const SOURCE_CONTROL_SSH_FOLDER = 'ssh';
export const SOURCE_CONTROL_SSH_KEY_NAME = 'key';
export const SOURCE_CONTROL_DEFAULT_BRANCH = 'main';
@@ -14,3 +14,5 @@ export const SOURCE_CONTROL_API_ROOT = 'source-control';
export const SOURCE_CONTROL_README = `
# n8n Source Control
`;
export const SOURCE_CONTROL_DEFAULT_NAME = 'n8n user';
export const SOURCE_CONTROL_DEFAULT_EMAIL = 'n8n@example.com';

106
packages/cli/src/environments/sourceControl/sourceControl.controller.ee.ts
View File

@@ -1,6 +1,3 @@
import express from 'express';
import { Service } from 'typedi';
import type { PullResult, PushResult, StatusResult } from 'simple-git';
import { Authorized, Get, Post, Patch, RestController } from '@/decorators';
import {
sourceControlLicensedMiddleware,
@@ -8,12 +5,18 @@ import {
} from './middleware/sourceControlEnabledMiddleware.ee';
import { SourceControlService } from './sourceControl.service.ee';
import { SourceControlRequest } from './types/requests';
import type { SourceControlPreferences } from './types/sourceControlPreferences';
import { BadRequestError } from '@/ResponseHelper';
import type { ImportResult } from './types/importResult';
import { SourceControlPreferencesService } from './sourceControlPreferences.service.ee';
import type { SourceControlPreferences } from './types/sourceControlPreferences';
import type { SourceControlledFile } from './types/sourceControlledFile';
import { SOURCE_CONTROL_API_ROOT, SOURCE_CONTROL_DEFAULT_BRANCH } from './constants';
import { BadRequestError } from '@/ResponseHelper';
import type { PullResult } from 'simple-git';
import express from 'express';
import type { ImportResult } from './types/importResult';
import Container, { Service } from 'typedi';
import { InternalHooks } from '../../InternalHooks';
import { getRepoType } from './sourceControlHelper.ee';
import { SourceControlGetStatus } from './types/sourceControlGetStatus';
@Service()
@RestController(`/${SOURCE_CONTROL_API_ROOT}`)
@@ -23,7 +26,7 @@ export class SourceControlController {
private sourceControlPreferencesService: SourceControlPreferencesService,
) {}
@Authorized('any')
@Authorized('none')
@Get('/preferences', { middlewares: [sourceControlLicensedMiddleware] })
async getPreferences(): Promise<SourceControlPreferences> {
// returns the settings with the privateKey property redacted
@@ -56,14 +59,17 @@ export class SourceControlController {
);
if (sanitizedPreferences.initRepo === true) {
try {
await this.sourceControlService.initializeRepository({
...updatedPreferences,
branchName:
updatedPreferences.branchName === ''
? SOURCE_CONTROL_DEFAULT_BRANCH
: updatedPreferences.branchName,
initRepo: true,
});
await this.sourceControlService.initializeRepository(
{
...updatedPreferences,
branchName:
updatedPreferences.branchName === ''
? SOURCE_CONTROL_DEFAULT_BRANCH
: updatedPreferences.branchName,
initRepo: true,
},
req.user,
);
if (this.sourceControlPreferencesService.getPreferences().branchName !== '') {
await this.sourceControlPreferencesService.setPreferences({
connected: true,
@@ -76,7 +82,17 @@ export class SourceControlController {
}
}
await this.sourceControlService.init();
return this.sourceControlPreferencesService.getPreferences();
const resultingPreferences = this.sourceControlPreferencesService.getPreferences();
// #region Tracking Information
// located in controller so as to not call this multiple times when updating preferences
void Container.get(InternalHooks).onSourceControlSettingsUpdated({
branch_name: resultingPreferences.branchName,
connected: resultingPreferences.connected,
read_only_instance: resultingPreferences.branchReadOnly,
repo_type: getRepoType(resultingPreferences.repositoryUrl),
});
// #endregion
return resultingPreferences;
} catch (error) {
throw new BadRequestError((error as { message: string }).message);
}
@@ -92,8 +108,6 @@ export class SourceControlController {
connected: undefined,
publicKey: undefined,
repositoryUrl: undefined,
authorName: undefined,
authorEmail: undefined,
};
const currentPreferences = this.sourceControlPreferencesService.getPreferences();
await this.sourceControlPreferencesService.validateSourceControlPreferences(
@@ -115,7 +129,14 @@ export class SourceControlController {
);
}
await this.sourceControlService.init();
return this.sourceControlPreferencesService.getPreferences();
const resultingPreferences = this.sourceControlPreferencesService.getPreferences();
void Container.get(InternalHooks).onSourceControlSettingsUpdated({
branch_name: resultingPreferences.branchName,
connected: resultingPreferences.connected,
read_only_instance: resultingPreferences.branchReadOnly,
repo_type: getRepoType(resultingPreferences.repositoryUrl),
});
return resultingPreferences;
} catch (error) {
throw new BadRequestError((error as { message: string }).message);
}
@@ -146,18 +167,18 @@ export class SourceControlController {
async pushWorkfolder(
req: SourceControlRequest.PushWorkFolder,
res: express.Response,
): Promise<PushResult | SourceControlledFile[]> {
): Promise<SourceControlledFile[]> {
if (this.sourceControlPreferencesService.isBranchReadOnly()) {
throw new BadRequestError('Cannot push onto read-only branch.');
}
try {
await this.sourceControlService.setGitUserDetails(
`${req.user.firstName} ${req.user.lastName}`,
req.user.email,
);
const result = await this.sourceControlService.pushWorkfolder(req.body);
if ((result as PushResult).pushed) {
res.statusCode = 200;
} else {
res.statusCode = 409;
}
return result;
res.statusCode = result.statusCode;
return result.statusResult;
} catch (error) {
throw new BadRequestError((error as { message: string }).message);
}
@@ -168,20 +189,15 @@ export class SourceControlController {
async pullWorkfolder(
req: SourceControlRequest.PullWorkFolder,
res: express.Response,
): Promise<SourceControlledFile[] | ImportResult | PullResult | StatusResult | undefined> {
): Promise<SourceControlledFile[] | ImportResult | PullResult | undefined> {
try {
const result = await this.sourceControlService.pullWorkfolder({
force: req.body.force,
variables: req.body.variables,
userId: req.user.id,
importAfterPull: req.body.importAfterPull ?? true,
});
if ((result as ImportResult)?.workflows) {
res.statusCode = 200;
} else {
res.statusCode = 409;
}
return result;
res.statusCode = result.statusCode;
return result.statusResult;
} catch (error) {
throw new BadRequestError((error as { message: string }).message);
}
@@ -189,16 +205,9 @@ export class SourceControlController {
@Authorized(['global', 'owner'])
@Get('/reset-workfolder', { middlewares: [sourceControlLicensedAndEnabledMiddleware] })
async resetWorkfolder(
req: SourceControlRequest.PullWorkFolder,
): Promise<ImportResult | undefined> {
async resetWorkfolder(): Promise<ImportResult | undefined> {
try {
return await this.sourceControlService.resetWorkfolder({
force: req.body.force,
variables: req.body.variables,
userId: req.user.id,
importAfterPull: req.body.importAfterPull ?? true,
});
return await this.sourceControlService.resetWorkfolder();
} catch (error) {
throw new BadRequestError((error as { message: string }).message);
}
@@ -206,9 +215,12 @@ export class SourceControlController {
@Authorized('any')
@Get('/get-status', { middlewares: [sourceControlLicensedAndEnabledMiddleware] })
async getStatus() {
async getStatus(req: SourceControlRequest.GetStatus) {
try {
return await this.sourceControlService.getStatus();
const result = (await this.sourceControlService.getStatus(
new SourceControlGetStatus(req.query),
)) as SourceControlledFile[];
return result;
} catch (error) {
throw new BadRequestError((error as { message: string }).message);
}
@@ -216,9 +228,9 @@ export class SourceControlController {
@Authorized('any')
@Get('/status', { middlewares: [sourceControlLicensedMiddleware] })
async status(): Promise<StatusResult> {
async status(req: SourceControlRequest.GetStatus) {
try {
return await this.sourceControlService.status();
return await this.sourceControlService.getStatus(new SourceControlGetStatus(req.query));
} catch (error) {
throw new BadRequestError((error as { message: string }).message);
}

912
packages/cli/src/environments/sourceControl/sourceControl.service.ee.ts
View File

File diff suppressed because it is too large Load Diff

212
packages/cli/src/environments/sourceControl/sourceControlExport.service.ee.ts
View File

@@ -3,23 +3,28 @@ import path from 'path';
import {
SOURCE_CONTROL_CREDENTIAL_EXPORT_FOLDER,
SOURCE_CONTROL_GIT_FOLDER,
SOURCE_CONTROL_OWNERS_EXPORT_FILE,
SOURCE_CONTROL_TAGS_EXPORT_FILE,
SOURCE_CONTROL_VARIABLES_EXPORT_FILE,
SOURCE_CONTROL_WORKFLOW_EXPORT_FOLDER,
} from './constants';
import * as Db from '@/Db';
import glob from 'fast-glob';
import type { ICredentialDataDecryptedObject } from 'n8n-workflow';
import { LoggerProxy, jsonParse } from 'n8n-workflow';
import { writeFile as fsWriteFile, readFile as fsReadFile, rm as fsRm } from 'fs/promises';
import { LoggerProxy } from 'n8n-workflow';
import { writeFile as fsWriteFile, rm as fsRm } from 'fs/promises';
import { rmSync } from 'fs';
import { Credentials, UserSettings } from 'n8n-core';
import type { IWorkflowToImport } from '@/Interfaces';
import type { ExportableWorkflow } from './types/exportableWorkflow';
import type { ExportableCredential } from './types/exportableCredential';
import type { ExportResult } from './types/exportResult';
import type { SharedWorkflow } from '@db/entities/SharedWorkflow';
import { sourceControlFoldersExistCheck } from './sourceControlHelper.ee';
import {
getCredentialExportPath,
getVariablesPath,
getWorkflowExportPath,
sourceControlFoldersExistCheck,
stringContainsExpression,
} from './sourceControlHelper.ee';
import type { WorkflowEntity } from '@db/entities/WorkflowEntity';
import { In } from 'typeorm';
import type { SourceControlledFile } from './types/sourceControlledFile';
@Service()
export class SourceControlExportService {
@@ -40,79 +45,11 @@ export class SourceControlExportService {
}
getWorkflowPath(workflowId: string): string {
return path.join(this.workflowExportFolder, `${workflowId}.json`);
return getWorkflowExportPath(workflowId, this.workflowExportFolder);
}
getCredentialsPath(credentialsId: string): string {
return path.join(this.credentialExportFolder, `${credentialsId}.json`);
}
getTagsPath(): string {
return path.join(this.gitFolder, SOURCE_CONTROL_TAGS_EXPORT_FILE);
}
getOwnersPath(): string {
return path.join(this.gitFolder, SOURCE_CONTROL_OWNERS_EXPORT_FILE);
}
getVariablesPath(): string {
return path.join(this.gitFolder, SOURCE_CONTROL_VARIABLES_EXPORT_FILE);
}
async getWorkflowFromFile(
filePath: string,
root = this.gitFolder,
): Promise<IWorkflowToImport | undefined> {
try {
const importedWorkflow = jsonParse<IWorkflowToImport>(
await fsReadFile(path.join(root, filePath), { encoding: 'utf8' }),
);
return importedWorkflow;
} catch (error) {
return undefined;
}
}
async getCredentialFromFile(
filePath: string,
root = this.gitFolder,
): Promise<ExportableCredential | undefined> {
try {
const credential = jsonParse<ExportableCredential>(
await fsReadFile(path.join(root, filePath), { encoding: 'utf8' }),
);
return credential;
} catch (error) {
return undefined;
}
}
async cleanWorkFolder() {
try {
const workflowFiles = await glob('*.json', {
cwd: this.workflowExportFolder,
absolute: true,
});
const credentialFiles = await glob('*.json', {
cwd: this.credentialExportFolder,
absolute: true,
});
const variablesFile = await glob(SOURCE_CONTROL_VARIABLES_EXPORT_FILE, {
cwd: this.gitFolder,
absolute: true,
});
const tagsFile = await glob(SOURCE_CONTROL_TAGS_EXPORT_FILE, {
cwd: this.gitFolder,
absolute: true,
});
await Promise.all(tagsFile.map(async (e) => fsRm(e)));
await Promise.all(variablesFile.map(async (e) => fsRm(e)));
await Promise.all(workflowFiles.map(async (e) => fsRm(e)));
await Promise.all(credentialFiles.map(async (e) => fsRm(e)));
LoggerProxy.debug('Cleaned work folder.');
} catch (error) {
LoggerProxy.error(`Failed to clean work folder: ${(error as Error).message}`);
}
return getCredentialExportPath(credentialsId, this.credentialExportFolder);
}
async deleteRepositoryFolder() {
@@ -123,86 +60,73 @@ export class SourceControlExportService {
}
}
private async rmDeletedWorkflowsFromExportFolder(
workflowsToBeExported: SharedWorkflow[],
): Promise<Set<string>> {
const sharedWorkflowsFileNames = new Set<string>(
workflowsToBeExported.map((e) => this.getWorkflowPath(e?.workflow?.name)),
);
const existingWorkflowsInFolder = new Set<string>(
await glob('*.json', {
cwd: this.workflowExportFolder,
absolute: true,
}),
);
const deletedWorkflows = new Set(existingWorkflowsInFolder);
for (const elem of sharedWorkflowsFileNames) {
deletedWorkflows.delete(elem);
}
public rmFilesFromExportFolder(filesToBeDeleted: Set<string>): Set<string> {
try {
await Promise.all([...deletedWorkflows].map(async (e) => fsRm(e)));
filesToBeDeleted.forEach((e) => rmSync(e));
} catch (error) {
LoggerProxy.error(`Failed to delete workflows from work folder: ${(error as Error).message}`);
}
return deletedWorkflows;
return filesToBeDeleted;
}
private async writeExportableWorkflowsToExportFolder(workflowsToBeExported: SharedWorkflow[]) {
private async writeExportableWorkflowsToExportFolder(
workflowsToBeExported: WorkflowEntity[],
owners: Record<string, string>,
) {
await Promise.all(
workflowsToBeExported.map(async (e) => {
if (!e.workflow) {
LoggerProxy.debug(
`Found no corresponding workflow ${e.workflowId ?? 'unknown'}, skipping export`,
);
return;
}
const fileName = this.getWorkflowPath(e.workflow?.id);
const fileName = this.getWorkflowPath(e.id);
const sanitizedWorkflow: ExportableWorkflow = {
active: e.workflow?.active,
id: e.workflow?.id,
name: e.workflow?.name,
nodes: e.workflow?.nodes,
connections: e.workflow?.connections,
settings: e.workflow?.settings,
triggerCount: e.workflow?.triggerCount,
versionId: e.workflow?.versionId,
id: e.id,
name: e.name,
nodes: e.nodes,
connections: e.connections,
settings: e.settings,
triggerCount: e.triggerCount,
versionId: e.versionId,
owner: owners[e.id],
};
LoggerProxy.debug(`Writing workflow ${e.workflowId} to ${fileName}`);
LoggerProxy.debug(`Writing workflow ${e.id} to ${fileName}`);
return fsWriteFile(fileName, JSON.stringify(sanitizedWorkflow, null, 2));
}),
);
}
async exportWorkflowsToWorkFolder(): Promise<ExportResult> {
async exportWorkflowsToWorkFolder(candidates: SourceControlledFile[]): Promise<ExportResult> {
try {
sourceControlFoldersExistCheck([this.workflowExportFolder]);
const workflowIds = candidates.map((e) => e.id);
const sharedWorkflows = await Db.collections.SharedWorkflow.find({
relations: ['workflow', 'role', 'user'],
relations: ['role', 'user'],
where: {
role: {
name: 'owner',
scope: 'workflow',
},
workflowId: In(workflowIds),
},
});
const workflows = await Db.collections.Workflow.find({
where: {
id: In(workflowIds),
},
});
// before exporting, figure out which workflows have been deleted and remove them from the export folder
const removedFiles = await this.rmDeletedWorkflowsFromExportFolder(sharedWorkflows);
// write the workflows to the export folder as json files
await this.writeExportableWorkflowsToExportFolder(sharedWorkflows);
// write list of owners to file
const ownersFileName = this.getOwnersPath();
// determine owner of each workflow to be exported
const owners: Record<string, string> = {};
sharedWorkflows.forEach((e) => (owners[e.workflowId] = e.user.email));
await fsWriteFile(ownersFileName, JSON.stringify(owners, null, 2));
// write the workflows to the export folder as json files
await this.writeExportableWorkflowsToExportFolder(workflows, owners);
// await fsWriteFile(ownersFileName, JSON.stringify(owners, null, 2));
return {
count: sharedWorkflows.length,
folder: this.workflowExportFolder,
files: sharedWorkflows.map((e) => ({
id: e?.workflow?.id,
name: this.getWorkflowPath(e?.workflow?.name),
files: workflows.map((e) => ({
id: e?.id,
name: this.getWorkflowPath(e?.name),
})),
removedFiles: [...removedFiles],
};
} catch (error) {
throw Error(`Failed to export workflows to work folder: ${(error as Error).message}`);
@@ -221,7 +145,7 @@ export class SourceControlExportService {
files: [],
};
}
const fileName = this.getVariablesPath();
const fileName = getVariablesPath(this.gitFolder);
const sanitizedVariables = variables.map((e) => ({ ...e, value: '' }));
await fsWriteFile(fileName, JSON.stringify(sanitizedVariables, null, 2));
return {
@@ -252,7 +176,7 @@ export class SourceControlExportService {
};
}
const mappings = await Db.collections.WorkflowTagMapping.find();
const fileName = this.getTagsPath();
const fileName = path.join(this.gitFolder, SOURCE_CONTROL_TAGS_EXPORT_FILE);
await fsWriteFile(
fileName,
JSON.stringify(
@@ -289,10 +213,7 @@ export class SourceControlExportService {
} else if (typeof data[key] === 'object') {
data[key] = this.replaceCredentialData(data[key] as ICredentialDataDecryptedObject);
} else if (typeof data[key] === 'string') {
data[key] =
(data[key] as string)?.startsWith('={{') && (data[key] as string)?.includes('$secret')
? data[key]
: '';
data[key] = stringContainsExpression(data[key] as string) ? data[key] : '';
} else if (typeof data[key] === 'number') {
// TODO: leaving numbers in for now, but maybe we should remove them
continue;
@@ -305,23 +226,31 @@ export class SourceControlExportService {
return data;
};
async exportCredentialsToWorkFolder(): Promise<ExportResult> {
async exportCredentialsToWorkFolder(candidates: SourceControlledFile[]): Promise<ExportResult> {
try {
sourceControlFoldersExistCheck([this.credentialExportFolder]);
const sharedCredentials = await Db.collections.SharedCredentials.find({
const credentialIds = candidates.map((e) => e.id);
const credentialsToBeExported = await Db.collections.SharedCredentials.find({
relations: ['credentials', 'role', 'user'],
where: {
credentialsId: In(credentialIds),
},
});
let missingIds: string[] = [];
if (credentialsToBeExported.length !== credentialIds.length) {
const foundCredentialIds = credentialsToBeExported.map((e) => e.credentialsId);
missingIds = credentialIds.filter(
(remote) => foundCredentialIds.findIndex((local) => local === remote) === -1,
);
}
const encryptionKey = await UserSettings.getEncryptionKey();
await Promise.all(
sharedCredentials.map(async (sharedCredential) => {
credentialsToBeExported.map(async (sharedCredential) => {
const { name, type, nodesAccess, data, id } = sharedCredential.credentials;
const credentialObject = new Credentials({ id, name }, type, nodesAccess, data);
const plainData = credentialObject.getData(encryptionKey);
const sanitizedData = this.replaceCredentialData(plainData);
const fileName = path.join(
this.credentialExportFolder,
`${sharedCredential.credentials.id}.json`,
);
const fileName = this.getCredentialsPath(sharedCredential.credentials.id);
const sanitizedCredential: ExportableCredential = {
id: sharedCredential.credentials.id,
name: sharedCredential.credentials.name,
@@ -334,12 +263,13 @@ export class SourceControlExportService {
}),
);
return {
count: sharedCredentials.length,
count: credentialsToBeExported.length,
folder: this.credentialExportFolder,
files: sharedCredentials.map((e) => ({
files: credentialsToBeExported.map((e) => ({
id: e.credentials.id,
name: path.join(this.credentialExportFolder, `${e.credentials.name}.json`),
})),
missingIds,
};
} catch (error) {
throw Error(`Failed to export credentials to work folder: ${(error as Error).message}`);

101
packages/cli/src/environments/sourceControl/sourceControlGit.service.ee.ts
View File

@@ -12,10 +12,16 @@ import type {
SimpleGitOptions,
StatusResult,
} from 'simple-git';
import { simpleGit } from 'simple-git';
import type { SourceControlPreferences } from './types/sourceControlPreferences';
import { SOURCE_CONTROL_DEFAULT_BRANCH, SOURCE_CONTROL_ORIGIN } from './constants';
import {
SOURCE_CONTROL_DEFAULT_BRANCH,
SOURCE_CONTROL_DEFAULT_EMAIL,
SOURCE_CONTROL_DEFAULT_NAME,
SOURCE_CONTROL_ORIGIN,
} from './constants';
import { sourceControlFoldersExistCheck } from './sourceControlHelper.ee';
import type { User } from '../../databases/entities/User';
import { getInstanceOwner } from '../../UserManagement/UserManagementHelper';
@Service()
export class SourceControlGitService {
@@ -27,7 +33,7 @@ export class SourceControlGitService {
* Run pre-checks before initialising git
* Checks for existence of required binaries (git and ssh)
*/
preInitCheck(): boolean {
private preInitCheck(): boolean {
LoggerProxy.debug('GitService.preCheck');
try {
const gitResult = execSync('git --version', {
@@ -80,6 +86,8 @@ export class SourceControlGitService {
trimmed: false,
};
const { simpleGit } = await import('simple-git');
this.git = simpleGit(this.gitOptions)
// Tell git not to ask for any information via the terminal like for
// example the username. As nobody will be able to answer it would
@@ -92,7 +100,8 @@ export class SourceControlGitService {
}
if (!(await this.hasRemote(sourceControlPreferences.repositoryUrl))) {
if (sourceControlPreferences.connected && sourceControlPreferences.repositoryUrl) {
await this.initRepository(sourceControlPreferences);
const user = await getInstanceOwner();
await this.initRepository(sourceControlPreferences, user);
}
}
}
@@ -101,9 +110,9 @@ export class SourceControlGitService {
this.git = null;
}
async checkRepositorySetup(): Promise<boolean> {
private async checkRepositorySetup(): Promise<boolean> {
if (!this.git) {
throw new Error('Git is not initialized');
throw new Error('Git is not initialized (async)');
}
if (!(await this.git.checkIsRepo())) {
return false;
@@ -116,9 +125,9 @@ export class SourceControlGitService {
}
}
async hasRemote(remote: string): Promise<boolean> {
private async hasRemote(remote: string): Promise<boolean> {
if (!this.git) {
throw new Error('Git is not initialized');
throw new Error('Git is not initialized (async)');
}
try {
const remotes = await this.git.getRemotes(true);
@@ -139,11 +148,12 @@ export class SourceControlGitService {
async initRepository(
sourceControlPreferences: Pick<
SourceControlPreferences,
'repositoryUrl' | 'authorEmail' | 'authorName' | 'branchName' | 'initRepo'
'repositoryUrl' | 'branchName' | 'initRepo'
>,
user: User,
): Promise<void> {
if (!this.git) {
throw new Error('Git is not initialized');
throw new Error('Git is not initialized (Promise)');
}
if (sourceControlPreferences.initRepo) {
try {
@@ -161,8 +171,10 @@ export class SourceControlGitService {
throw error;
}
}
await this.git.addConfig('user.email', sourceControlPreferences.authorEmail);
await this.git.addConfig('user.name', sourceControlPreferences.authorName);
await this.setGitUserDetails(
`${user.firstName} ${user.lastName}` ?? SOURCE_CONTROL_DEFAULT_NAME,
user.email ?? SOURCE_CONTROL_DEFAULT_EMAIL,
);
if (sourceControlPreferences.initRepo) {
try {
const branches = await this.getBranches();
@@ -175,9 +187,17 @@ export class SourceControlGitService {
}
}
async setGitUserDetails(name: string, email: string): Promise<void> {
if (!this.git) {
throw new Error('Git is not initialized (setGitUserDetails)');
}
await this.git.addConfig('user.email', name);
await this.git.addConfig('user.name', email);
}
async getBranches(): Promise<{ branches: string[]; currentBranch: string }> {
if (!this.git) {
throw new Error('Git is not initialized');
throw new Error('Git is not initialized (getBranches)');
}
try {
@@ -200,23 +220,16 @@ export class SourceControlGitService {
async setBranch(branch: string): Promise<{ branches: string[]; currentBranch: string }> {
if (!this.git) {
throw new Error('Git is not initialized');
throw new Error('Git is not initialized (setBranch)');
}
await this.git.checkout(branch);
await this.git.branch([`--set-upstream-to=${SOURCE_CONTROL_ORIGIN}/${branch}`, branch]);
return this.getBranches();
}
async fetch(): Promise<FetchResult> {
if (!this.git) {
throw new Error('Git is not initialized');
}
return this.git.fetch();
}
async getCurrentBranch(): Promise<{ current: string; remote: string }> {
if (!this.git) {
throw new Error('Git is not initialized');
throw new Error('Git is not initialized (getCurrentBranch)');
}
const currentBranch = (await this.git.branch()).current;
return {
@@ -225,49 +238,47 @@ export class SourceControlGitService {
};
}
async diff(options?: { target?: string; dots?: '..' | '...' }): Promise<DiffResult> {
if (!this.git) {
throw new Error('Git is not initialized');
}
const currentBranch = await this.getCurrentBranch();
const target = options?.target ?? currentBranch.remote;
const dots = options?.dots ?? '...';
return this.git.diffSummary([dots + target]);
}
async diffRemote(): Promise<DiffResult | undefined> {
if (!this.git) {
throw new Error('Git is not initialized');
throw new Error('Git is not initialized (diffRemote)');
}
const currentBranch = await this.getCurrentBranch();
if (currentBranch.remote) {
const target = currentBranch.remote;
return this.git.diffSummary(['...' + target]);
return this.git.diffSummary(['...' + target, '--ignore-all-space']);
}
return;
}
async diffLocal(): Promise<DiffResult | undefined> {
if (!this.git) {
throw new Error('Git is not initialized');
throw new Error('Git is not initialized (diffLocal)');
}
const currentBranch = await this.getCurrentBranch();
if (currentBranch.remote) {
const target = currentBranch.current;
return this.git.diffSummary([target]);
return this.git.diffSummary([target, '--ignore-all-space']);
}
return;
}
async fetch(): Promise<FetchResult> {
if (!this.git) {
throw new Error('Git is not initialized (fetch)');
}
return this.git.fetch();
}
async pull(options: { ffOnly: boolean } = { ffOnly: true }): Promise<PullResult> {
if (!this.git) {
throw new Error('Git is not initialized');
throw new Error('Git is not initialized (pull)');
}
const params = {};
if (options.ffOnly) {
// eslint-disable-next-line @typescript-eslint/naming-convention
return this.git.pull(undefined, undefined, { '--ff-only': null });
Object.assign(params, { '--ff-only': true });
}
return this.git.pull();
return this.git.pull(params);
}
async push(
@@ -278,7 +289,7 @@ export class SourceControlGitService {
): Promise<PushResult> {
const { force, branch } = options;
if (!this.git) {
throw new Error('Git is not initialized');
throw new Error('Git is not initialized ({)');
}
if (force) {
return this.git.push(SOURCE_CONTROL_ORIGIN, branch, ['-f']);
@@ -288,7 +299,7 @@ export class SourceControlGitService {
async stage(files: Set<string>, deletedFiles?: Set<string>): Promise<string> {
if (!this.git) {
throw new Error('Git is not initialized');
throw new Error('Git is not initialized (stage)');
}
if (deletedFiles?.size) {
try {
@@ -301,10 +312,10 @@ export class SourceControlGitService {
}
async resetBranch(
options: { hard?: boolean; target: string } = { hard: false, target: 'HEAD' },
options: { hard: boolean; target: string } = { hard: true, target: 'HEAD' },
): Promise<string> {
if (!this.git) {
throw new Error('Git is not initialized');
throw new Error('Git is not initialized (Promise)');
}
if (options?.hard) {
return this.git.raw(['reset', '--hard', options.target]);
@@ -316,14 +327,14 @@ export class SourceControlGitService {
async commit(message: string): Promise<CommitResult> {
if (!this.git) {
throw new Error('Git is not initialized');
throw new Error('Git is not initialized (commit)');
}
return this.git.commit(message);
}
async status(): Promise<StatusResult> {
if (!this.git) {
throw new Error('Git is not initialized');
throw new Error('Git is not initialized (status)');
}
const statusResult = await this.git.status();
return statusResult;

132
packages/cli/src/environments/sourceControl/sourceControlHelper.ee.ts
View File

@@ -1,25 +1,61 @@
import { Container } from 'typedi';
import Container from 'typedi';
import { License } from '@/License';
import { generateKeyPairSync } from 'crypto';
import sshpk from 'sshpk';
import type { KeyPair } from './types/keyPair';
import { constants as fsConstants, mkdirSync, accessSync } from 'fs';
import { LoggerProxy } from 'n8n-workflow';
import { License } from '@/License';
import type { KeyPair } from './types/keyPair';
import { SOURCE_CONTROL_GIT_KEY_COMMENT } from './constants';
import {
SOURCE_CONTROL_GIT_KEY_COMMENT,
SOURCE_CONTROL_TAGS_EXPORT_FILE,
SOURCE_CONTROL_VARIABLES_EXPORT_FILE,
} from './constants';
import type { SourceControlledFile } from './types/sourceControlledFile';
import path from 'path';
export function sourceControlFoldersExistCheck(folders: string[]) {
export function stringContainsExpression(testString: string): boolean {
return /^=.*\{\{.*\}\}/.test(testString);
}
export function getWorkflowExportPath(workflowId: string, workflowExportFolder: string): string {
return path.join(workflowExportFolder, `${workflowId}.json`);
}
export function getCredentialExportPath(
credentialId: string,
credentialExportFolder: string,
): string {
return path.join(credentialExportFolder, `${credentialId}.json`);
}
export function getVariablesPath(gitFolder: string): string {
return path.join(gitFolder, SOURCE_CONTROL_VARIABLES_EXPORT_FILE);
}
export function getTagsPath(gitFolder: string): string {
return path.join(gitFolder, SOURCE_CONTROL_TAGS_EXPORT_FILE);
}
export function sourceControlFoldersExistCheck(
folders: string[],
createIfNotExists = true,
): boolean {
// running these file access function synchronously to avoid race conditions
let existed = true;
folders.forEach((folder) => {
try {
accessSync(folder, fsConstants.F_OK);
} catch {
try {
mkdirSync(folder);
} catch (error) {
LoggerProxy.error((error as Error).message);
existed = false;
if (createIfNotExists) {
try {
mkdirSync(folder, { recursive: true });
} catch (error) {
LoggerProxy.error((error as Error).message);
}
}
}
});
return existed;
}
export function isSourceControlLicensed() {
@@ -27,7 +63,8 @@ export function isSourceControlLicensed() {
return license.isSourceControlLicensed();
}
export function generateSshKeyPair(keyType: 'ed25519' | 'rsa' = 'ed25519') {
export async function generateSshKeyPair(keyType: 'ed25519' | 'rsa' = 'ed25519') {
const sshpk = await import('sshpk');
const keyPair: KeyPair = {
publicKey: '',
privateKey: '',
@@ -65,3 +102,76 @@ export function generateSshKeyPair(keyType: 'ed25519' | 'rsa' = 'ed25519') {
publicKey: keyPair.publicKey,
};
}
export function getRepoType(repoUrl: string): 'github' | 'gitlab' | 'other' {
if (repoUrl.includes('github.com')) {
return 'github';
} else if (repoUrl.includes('gitlab.com')) {
return 'gitlab';
}
return 'other';
}
function filterSourceControlledFilesUniqueIds(files: SourceControlledFile[]) {
return (
files.filter((file, index, self) => {
return self.findIndex((f) => f.id === file.id) === index;
}) || []
);
}
export function getTrackingInformationFromPullResult(result: SourceControlledFile[]): {
cred_conflicts: number;
workflow_conflicts: number;
workflow_updates: number;
} {
const uniques = filterSourceControlledFilesUniqueIds(result);
return {
cred_conflicts: uniques.filter(
(file) =>
file.type === 'credential' && file.status === 'modified' && file.location === 'local',
).length,
workflow_conflicts: uniques.filter(
(file) => file.type === 'workflow' && file.status === 'modified' && file.location === 'local',
).length,
workflow_updates: uniques.filter((file) => file.type === 'workflow').length,
};
}
export function getTrackingInformationFromPrePushResult(result: SourceControlledFile[]): {
workflows_eligible: number;
workflows_eligible_with_conflicts: number;
creds_eligible: number;
creds_eligible_with_conflicts: number;
variables_eligible: number;
} {
const uniques = filterSourceControlledFilesUniqueIds(result);
return {
workflows_eligible: uniques.filter((file) => file.type === 'workflow').length,
workflows_eligible_with_conflicts: uniques.filter(
(file) => file.type === 'workflow' && file.conflict,
).length,
creds_eligible: uniques.filter((file) => file.type === 'credential').length,
creds_eligible_with_conflicts: uniques.filter(
(file) => file.type === 'credential' && file.conflict,
).length,
variables_eligible: uniques.filter((file) => file.type === 'variables').length,
};
}
export function getTrackingInformationFromPostPushResult(result: SourceControlledFile[]): {
workflows_eligible: number;
workflows_pushed: number;
creds_pushed: number;
variables_pushed: number;
} {
const uniques = filterSourceControlledFilesUniqueIds(result);
return {
workflows_pushed: uniques.filter((file) => file.pushed && file.type === 'workflow').length ?? 0,
workflows_eligible: uniques.filter((file) => file.type === 'workflow').length ?? 0,
creds_pushed:
uniques.filter((file) => file.pushed && file.file.startsWith('credential_stubs')).length ?? 0,
variables_pushed:
uniques.filter((file) => file.pushed && file.file.startsWith('variable_stubs')).length ?? 0,
};
}

503
packages/cli/src/environments/sourceControl/sourceControlImport.service.ee.ts
View File

@@ -3,7 +3,6 @@ import path from 'path';
import {
SOURCE_CONTROL_CREDENTIAL_EXPORT_FOLDER,
SOURCE_CONTROL_GIT_FOLDER,
SOURCE_CONTROL_OWNERS_EXPORT_FILE,
SOURCE_CONTROL_TAGS_EXPORT_FILE,
SOURCE_CONTROL_VARIABLES_EXPORT_FILE,
SOURCE_CONTROL_WORKFLOW_EXPORT_FOLDER,
@@ -16,15 +15,16 @@ import { Credentials, UserSettings } from 'n8n-core';
import type { IWorkflowToImport } from '@/Interfaces';
import type { ExportableCredential } from './types/exportableCredential';
import { Variables } from '@db/entities/Variables';
import type { ImportResult } from './types/importResult';
import { UM_FIX_INSTRUCTION } from '@/commands/BaseCommand';
import { SharedCredentials } from '@db/entities/SharedCredentials';
import type { WorkflowTagMapping } from '@db/entities/WorkflowTagMapping';
import type { TagEntity } from '@db/entities/TagEntity';
import { ActiveWorkflowRunner } from '../../ActiveWorkflowRunner';
import type { SourceControllPullOptions } from './types/sourceControlPullWorkFolder';
import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
import { In } from 'typeorm';
import { isUniqueConstraintError } from '../../ResponseHelper';
import { isUniqueConstraintError } from '@/ResponseHelper';
import type { SourceControlWorkflowVersionId } from './types/sourceControlWorkflowVersionId';
import { getCredentialExportPath, getWorkflowExportPath } from './sourceControlHelper.ee';
import type { SourceControlledFile } from './types/sourceControlledFile';
@Service()
export class SourceControlImportService {
@@ -143,65 +143,113 @@ export class SourceControlImportService {
return importCredentialsResult.filter((e) => e !== undefined);
}
private async importVariablesFromFile(valueOverrides?: {
[key: string]: string;
}): Promise<{ imported: string[] }> {
public async getRemoteVersionIdsFromFiles(): Promise<SourceControlWorkflowVersionId[]> {
const remoteWorkflowFiles = await glob('*.json', {
cwd: this.workflowExportFolder,
absolute: true,
});
const remoteWorkflowFilesParsed = await Promise.all(
remoteWorkflowFiles.map(async (file) => {
LoggerProxy.debug(`Parsing workflow file ${file}`);
const remote = jsonParse<IWorkflowToImport>(await fsReadFile(file, { encoding: 'utf8' }));
if (!remote?.id) {
return undefined;
}
return {
id: remote.id,
versionId: remote.versionId,
name: remote.name,
remoteId: remote.id,
filename: getWorkflowExportPath(remote.id, this.workflowExportFolder),
} as SourceControlWorkflowVersionId;
}),
);
return remoteWorkflowFilesParsed.filter(
(e) => e !== undefined,
) as SourceControlWorkflowVersionId[];
}
public async getLocalVersionIdsFromDb(): Promise<SourceControlWorkflowVersionId[]> {
const localWorkflows = await Db.collections.Workflow.find({
select: ['id', 'name', 'versionId', 'updatedAt'],
});
return localWorkflows.map((local) => ({
id: local.id,
versionId: local.versionId,
name: local.name,
localId: local.id,
filename: getWorkflowExportPath(local.id, this.workflowExportFolder),
updatedAt: local.updatedAt.toISOString(),
})) as SourceControlWorkflowVersionId[];
}
public async getRemoteCredentialsFromFiles(): Promise<
Array<ExportableCredential & { filename: string }>
> {
const remoteCredentialFiles = await glob('*.json', {
cwd: this.credentialExportFolder,
absolute: true,
});
const remoteCredentialFilesParsed = await Promise.all(
remoteCredentialFiles.map(async (file) => {
LoggerProxy.debug(`Parsing credential file ${file}`);
const remote = jsonParse<ExportableCredential>(
await fsReadFile(file, { encoding: 'utf8' }),
);
if (!remote?.id) {
return undefined;
}
return {
...remote,
filename: getCredentialExportPath(remote.id, this.credentialExportFolder),
};
}),
);
return remoteCredentialFilesParsed.filter((e) => e !== undefined) as Array<
ExportableCredential & { filename: string }
>;
}
public async getLocalCredentialsFromDb(): Promise<
Array<ExportableCredential & { filename: string }>
> {
const localCredentials = await Db.collections.Credentials.find({
select: ['id', 'name', 'type', 'nodesAccess'],
});
return localCredentials.map((local) => ({
id: local.id,
name: local.name,
type: local.type,
nodesAccess: local.nodesAccess,
filename: getCredentialExportPath(local.id, this.credentialExportFolder),
})) as Array<ExportableCredential & { filename: string }>;
}
public async getRemoteVariablesFromFile(): Promise<Variables[]> {
const variablesFile = await glob(SOURCE_CONTROL_VARIABLES_EXPORT_FILE, {
cwd: this.gitFolder,
absolute: true,
});
const result: { imported: string[] } = { imported: [] };
if (variablesFile.length > 0) {
LoggerProxy.debug(`Importing variables from file ${variablesFile[0]}`);
const importedVariables = jsonParse<Array<Partial<Variables>>>(
await fsReadFile(variablesFile[0], { encoding: 'utf8' }),
{ fallbackValue: [] },
);
const overriddenKeys = Object.keys(valueOverrides ?? {});
for (const variable of importedVariables) {
if (!variable.key) {
continue;
}
// by default no value is stored remotely, so an empty string is retuned
// it must be changed to undefined so as to not overwrite existing values!
if (variable.value === '') {
variable.value = undefined;
}
if (overriddenKeys.includes(variable.key) && valueOverrides) {
variable.value = valueOverrides[variable.key];
overriddenKeys.splice(overriddenKeys.indexOf(variable.key), 1);
}
try {
await Db.collections.Variables.upsert({ ...variable }, ['id']);
} catch (errorUpsert) {
if (isUniqueConstraintError(errorUpsert as Error)) {
LoggerProxy.debug(`Variable ${variable.key} already exists, updating instead`);
try {
await Db.collections.Variables.update({ key: variable.key }, { ...variable });
} catch (errorUpdate) {
LoggerProxy.debug(`Failed to update variable ${variable.key}, skipping`);
LoggerProxy.debug((errorUpdate as Error).message);
}
}
} finally {
result.imported.push(variable.key);
}
}
// add remaining overrides as new variables
if (overriddenKeys.length > 0 && valueOverrides) {
for (const key of overriddenKeys) {
result.imported.push(key);
const newVariable = new Variables({ key, value: valueOverrides[key] });
await Db.collections.Variables.save(newVariable);
}
}
return jsonParse<Variables[]>(await fsReadFile(variablesFile[0], { encoding: 'utf8' }), {
fallbackValue: [],
});
}
return result;
return [];
}
private async importTagsFromFile() {
public async getLocalVariablesFromDb(): Promise<Variables[]> {
const localVariables = await Db.collections.Variables.find({
select: ['id', 'key', 'type', 'value'],
});
return localVariables;
}
public async getRemoteTagsAndMappingsFromFile(): Promise<{
tags: TagEntity[];
mappings: WorkflowTagMapping[];
}> {
const tagsFile = await glob(SOURCE_CONTROL_TAGS_EXPORT_FILE, {
cwd: this.gitFolder,
absolute: true,
@@ -212,110 +260,51 @@ export class SourceControlImportService {
await fsReadFile(tagsFile[0], { encoding: 'utf8' }),
{ fallbackValue: { tags: [], mappings: [] } },
);
const existingWorkflowIds = new Set(
(
await Db.collections.Workflow.find({
select: ['id'],
})
).map((e) => e.id),
);
await Promise.all(
mappedTags.tags.map(async (tag) => {
await Db.collections.Tag.upsert(
{
...tag,
},
{
skipUpdateIfNoValuesChanged: true,
conflictPaths: { id: true },
},
);
}),
);
await Promise.all(
mappedTags.mappings.map(async (mapping) => {
if (!existingWorkflowIds.has(String(mapping.workflowId))) return;
await Db.collections.WorkflowTagMapping.upsert(
{ tagId: String(mapping.tagId), workflowId: String(mapping.workflowId) },
{
skipUpdateIfNoValuesChanged: true,
conflictPaths: { tagId: true, workflowId: true },
},
);
}),
);
return mappedTags;
}
return { tags: [], mappings: [] };
}
private async importWorkflowsFromFiles(
userId: string,
): Promise<Array<{ id: string; name: string }>> {
const workflowFiles = await glob('*.json', {
cwd: this.workflowExportFolder,
absolute: true,
public async getLocalTagsAndMappingsFromDb(): Promise<{
tags: TagEntity[];
mappings: WorkflowTagMapping[];
}> {
const localTags = await Db.collections.Tag.find({
select: ['id', 'name'],
});
const existingWorkflows = await Db.collections.Workflow.find({
select: ['id', 'name', 'active', 'versionId'],
const localMappings = await Db.collections.WorkflowTagMapping.find({
select: ['workflowId', 'tagId'],
});
return { tags: localTags, mappings: localMappings };
}
public async importWorkflowFromWorkFolder(candidates: SourceControlledFile[], userId: string) {
const ownerWorkflowRole = await this.getOwnerWorkflowRole();
const workflowRunner = Container.get(ActiveWorkflowRunner);
// read owner file if it exists and map workflow ids to owner emails
// then find existing users with those emails or fallback to passed in userId
const ownerRecords: Record<string, string> = {};
const ownersFile = await glob(SOURCE_CONTROL_OWNERS_EXPORT_FILE, {
cwd: this.gitFolder,
absolute: true,
const candidateIds = candidates.map((c) => c.id);
const existingWorkflows = await Db.collections.Workflow.find({
where: {
id: In(candidateIds),
},
select: ['id', 'name', 'versionId', 'active'],
});
if (ownersFile.length > 0) {
LoggerProxy.debug(`Reading workflow owners from file ${ownersFile[0]}`);
const ownerEmails = jsonParse<Record<string, string>>(
await fsReadFile(ownersFile[0], { encoding: 'utf8' }),
{ fallbackValue: {} },
);
if (ownerEmails) {
const uniqueOwnerEmails = new Set(Object.values(ownerEmails));
const existingUsers = await Db.collections.User.find({
where: { email: In([...uniqueOwnerEmails]) },
});
Object.keys(ownerEmails).forEach((workflowId) => {
ownerRecords[workflowId] =
existingUsers.find((e) => e.email === ownerEmails[workflowId])?.id ?? userId;
});
}
}
let importWorkflowsResult = new Array<{ id: string; name: string } | undefined>();
const allSharedWorkflows = await Db.collections.SharedWorkflow.find({
where: {
workflowId: In(candidateIds),
},
select: ['workflowId', 'roleId', 'userId'],
});
importWorkflowsResult = await Promise.all(
workflowFiles.map(async (file) => {
LoggerProxy.debug(`Parsing workflow file ${file}`);
const importedWorkflow = jsonParse<IWorkflowToImport>(
await fsReadFile(file, { encoding: 'utf8' }),
const cachedOwnerIds = new Map<string, string>();
const importWorkflowsResult = await Promise.all(
candidates.map(async (candidate) => {
LoggerProxy.debug(`Parsing workflow file ${candidate.file}`);
const importedWorkflow = jsonParse<IWorkflowToImport & { owner: string }>(
await fsReadFile(candidate.file, { encoding: 'utf8' }),
);
if (!importedWorkflow?.id) {
return;
}
const existingWorkflow = existingWorkflows.find((e) => e.id === importedWorkflow.id);
if (existingWorkflow?.versionId === importedWorkflow.versionId) {
LoggerProxy.debug(
`Skipping import of workflow ${importedWorkflow.id ?? 'n/a'} - versionId is up to date`,
);
return {
id: importedWorkflow.id ?? 'n/a',
name: 'skipped',
};
}
LoggerProxy.debug(`Importing workflow ${importedWorkflow.id ?? 'n/a'}`);
importedWorkflow.active = existingWorkflow?.active ?? false;
LoggerProxy.debug(`Updating workflow id ${importedWorkflow.id ?? 'new'}`);
const upsertResult = await Db.collections.Workflow.upsert({ ...importedWorkflow }, ['id']);
@@ -324,12 +313,31 @@ export class SourceControlImportService {
}
// Update workflow owner to the user who exported the workflow, if that user exists
// in the instance, and the workflow doesn't already have an owner
const workflowOwnerId = ownerRecords[importedWorkflow.id] ?? userId;
let workflowOwnerId = userId;
if (cachedOwnerIds.has(importedWorkflow.owner)) {
workflowOwnerId = cachedOwnerIds.get(importedWorkflow.owner) ?? userId;
} else {
const foundUser = await Db.collections.User.findOne({
where: {
email: importedWorkflow.owner,
},
select: ['id'],
});
if (foundUser) {
cachedOwnerIds.set(importedWorkflow.owner, foundUser.id);
workflowOwnerId = foundUser.id;
}
}
const existingSharedWorkflowOwnerByRoleId = allSharedWorkflows.find(
(e) => e.workflowId === importedWorkflow.id && e.roleId === ownerWorkflowRole.id,
(e) =>
e.workflowId === importedWorkflow.id &&
e.roleId.toString() === ownerWorkflowRole.id.toString(),
);
const existingSharedWorkflowOwnerByUserId = allSharedWorkflows.find(
(e) => e.workflowId === importedWorkflow.id && e.userId === workflowOwnerId,
(e) =>
e.workflowId === importedWorkflow.id &&
e.roleId.toString() === workflowOwnerId.toString(),
);
if (!existingSharedWorkflowOwnerByUserId && !existingSharedWorkflowOwnerByRoleId) {
// no owner exists yet, so create one
@@ -361,39 +369,218 @@ export class SourceControlImportService {
// try activating the imported workflow
LoggerProxy.debug(`Reactivating workflow id ${existingWorkflow.id}`);
await workflowRunner.add(existingWorkflow.id, 'activate');
// update the versionId of the workflow to match the imported workflow
} catch (error) {
LoggerProxy.error(`Failed to activate workflow ${existingWorkflow.id}`, error as Error);
} finally {
await Db.collections.Workflow.update(
{ id: existingWorkflow.id },
{ versionId: importedWorkflow.versionId },
);
}
}
return {
id: importedWorkflow.id ?? 'unknown',
name: file,
name: candidate.file,
};
}),
);
return importWorkflowsResult.filter((e) => e !== undefined) as Array<{
id: string;
name: string;
}>;
}
async importFromWorkFolder(options: SourceControllPullOptions): Promise<ImportResult> {
try {
const importedVariables = await this.importVariablesFromFile(options.variables);
const importedCredentials = await this.importCredentialsFromFiles(options.userId);
const importWorkflows = await this.importWorkflowsFromFiles(options.userId);
const importTags = await this.importTagsFromFile();
public async importCredentialsFromWorkFolder(candidates: SourceControlledFile[], userId: string) {
const candidateIds = candidates.map((c) => c.id);
const existingCredentials = await Db.collections.Credentials.find({
where: {
id: In(candidateIds),
},
select: ['id', 'name', 'type', 'data'],
});
const ownerCredentialRole = await this.getOwnerCredentialRole();
const ownerGlobalRole = await this.getOwnerGlobalRole();
const existingSharedCredentials = await Db.collections.SharedCredentials.find({
select: ['userId', 'credentialsId', 'roleId'],
where: {
credentialsId: In(candidateIds),
roleId: In([ownerCredentialRole.id, ownerGlobalRole.id]),
},
});
const encryptionKey = await UserSettings.getEncryptionKey();
let importCredentialsResult: Array<{ id: string; name: string; type: string }> = [];
importCredentialsResult = await Promise.all(
candidates.map(async (candidate) => {
LoggerProxy.debug(`Importing credentials file ${candidate.file}`);
const credential = jsonParse<ExportableCredential>(
await fsReadFile(candidate.file, { encoding: 'utf8' }),
);
const existingCredential = existingCredentials.find(
(e) => e.id === credential.id && e.type === credential.type,
);
const sharedOwner = existingSharedCredentials.find(
(e) => e.credentialsId === credential.id,
);
return {
variables: importedVariables,
credentials: importedCredentials,
workflows: importWorkflows,
tags: importTags,
};
const { name, type, data, id, nodesAccess } = credential;
const newCredentialObject = new Credentials({ id, name }, type, []);
if (existingCredential?.data) {
newCredentialObject.data = existingCredential.data;
} else {
newCredentialObject.setData(data, encryptionKey);
}
newCredentialObject.nodesAccess = nodesAccess || existingCredential?.nodesAccess || [];
LoggerProxy.debug(`Updating credential id ${newCredentialObject.id as string}`);
await Db.collections.Credentials.upsert(newCredentialObject, ['id']);
if (!sharedOwner) {
const newSharedCredential = new SharedCredentials();
newSharedCredential.credentialsId = newCredentialObject.id as string;
newSharedCredential.userId = userId;
newSharedCredential.roleId = ownerCredentialRole.id;
await Db.collections.SharedCredentials.upsert({ ...newSharedCredential }, [
'credentialsId',
'userId',
]);
}
return {
id: newCredentialObject.id as string,
name: newCredentialObject.name,
type: newCredentialObject.type,
};
}),
);
return importCredentialsResult.filter((e) => e !== undefined);
}
public async importTagsFromWorkFolder(candidate: SourceControlledFile) {
let mappedTags;
try {
LoggerProxy.debug(`Importing tags from file ${candidate.file}`);
mappedTags = jsonParse<{ tags: TagEntity[]; mappings: WorkflowTagMapping[] }>(
await fsReadFile(candidate.file, { encoding: 'utf8' }),
{ fallbackValue: { tags: [], mappings: [] } },
);
} catch (error) {
throw Error(`Failed to import workflows from work folder: ${(error as Error).message}`);
LoggerProxy.error(`Failed to import tags from file ${candidate.file}`, error as Error);
return;
}
if (mappedTags.mappings.length === 0 && mappedTags.tags.length === 0) {
return;
}
const existingWorkflowIds = new Set(
(
await Db.collections.Workflow.find({
select: ['id'],
})
).map((e) => e.id),
);
await Promise.all(
mappedTags.tags.map(async (tag) => {
const findByName = await Db.collections.Tag.findOne({
where: { name: tag.name },
select: ['id'],
});
if (findByName && findByName.id !== tag.id) {
throw new Error(
`A tag with the name <strong>${tag.name}</strong> already exists locally.<br />Please either rename the local tag, or the remote one with the id <strong>${tag.id}</strong> in the tags.json file.`,
);
}
await Db.collections.Tag.upsert(
{
...tag,
},
{
skipUpdateIfNoValuesChanged: true,
conflictPaths: { id: true },
},
);
}),
);
await Promise.all(
mappedTags.mappings.map(async (mapping) => {
if (!existingWorkflowIds.has(String(mapping.workflowId))) return;
await Db.collections.WorkflowTagMapping.upsert(
{ tagId: String(mapping.tagId), workflowId: String(mapping.workflowId) },
{
skipUpdateIfNoValuesChanged: true,
conflictPaths: { tagId: true, workflowId: true },
},
);
}),
);
return mappedTags;
}
public async importVariablesFromWorkFolder(
candidate: SourceControlledFile,
valueOverrides?: {
[key: string]: string;
},
) {
const result: { imported: string[] } = { imported: [] };
let importedVariables;
try {
LoggerProxy.debug(`Importing variables from file ${candidate.file}`);
importedVariables = jsonParse<Array<Partial<Variables>>>(
await fsReadFile(candidate.file, { encoding: 'utf8' }),
{ fallbackValue: [] },
);
} catch (error) {
LoggerProxy.error(`Failed to import tags from file ${candidate.file}`, error as Error);
return;
}
const overriddenKeys = Object.keys(valueOverrides ?? {});
for (const variable of importedVariables) {
if (!variable.key) {
continue;
}
// by default no value is stored remotely, so an empty string is retuned
// it must be changed to undefined so as to not overwrite existing values!
if (variable.value === '') {
variable.value = undefined;
}
if (overriddenKeys.includes(variable.key) && valueOverrides) {
variable.value = valueOverrides[variable.key];
overriddenKeys.splice(overriddenKeys.indexOf(variable.key), 1);
}
try {
await Db.collections.Variables.upsert({ ...variable }, ['id']);
} catch (errorUpsert) {
if (isUniqueConstraintError(errorUpsert as Error)) {
LoggerProxy.debug(`Variable ${variable.key} already exists, updating instead`);
try {
await Db.collections.Variables.update({ key: variable.key }, { ...variable });
} catch (errorUpdate) {
LoggerProxy.debug(`Failed to update variable ${variable.key}, skipping`);
LoggerProxy.debug((errorUpdate as Error).message);
}
}
} finally {
result.imported.push(variable.key);
}
}
// add remaining overrides as new variables
if (overriddenKeys.length > 0 && valueOverrides) {
for (const key of overriddenKeys) {
result.imported.push(key);
const newVariable = new Variables({ key, value: valueOverrides[key] });
await Db.collections.Variables.save(newVariable);
}
}
return result;
}
}

10
packages/cli/src/environments/sourceControl/sourceControlPreferences.service.ee.ts
View File

@@ -53,6 +53,14 @@ export class SourceControlPreferencesService {
);
}
public isSourceControlSetup() {
return (
this.isSourceControlLicensedAndEnabled() &&
this.getPreferences().repositoryUrl &&
this.getPreferences().branchName
);
}
getPublicKey(): string {
try {
return fsReadFileSync(this.sshKeyName + '.pub', { encoding: 'utf8' });
@@ -80,7 +88,7 @@ export class SourceControlPreferencesService {
*/
async generateAndSaveKeyPair(): Promise<SourceControlPreferences> {
sourceControlFoldersExistCheck([this.gitFolder, this.sshFolder]);
const keyPair = generateSshKeyPair('ed25519');
const keyPair = await generateSshKeyPair('ed25519');
if (keyPair.publicKey && keyPair.privateKey) {
try {
await fsWriteFile(this.sshKeyName + '.pub', keyPair.publicKey, {

1
packages/cli/src/environments/sourceControl/types/exportResult.ts
View File

@@ -6,4 +6,5 @@ export interface ExportResult {
name: string;
}>;
removedFiles?: string[];
missingIds?: string[];
}

2
packages/cli/src/environments/sourceControl/types/exportableWorkflow.ts
View File

@@ -1,7 +1,6 @@
import type { INode, IConnections, IWorkflowSettings } from 'n8n-workflow';
export interface ExportableWorkflow {
active: boolean;
id: string;
name: string;
nodes: INode[];
@@ -9,4 +8,5 @@ export interface ExportableWorkflow {
settings?: IWorkflowSettings;
triggerCount: number;
versionId: string;
owner: string;
}

2
packages/cli/src/environments/sourceControl/types/requests.ts
View File

@@ -8,6 +8,7 @@ import type { SourceControlPushWorkFolder } from './sourceControlPushWorkFolder'
import type { SourceControlPullWorkFolder } from './sourceControlPullWorkFolder';
import type { SourceControlDisconnect } from './sourceControlDisconnect';
import type { SourceControlSetReadOnly } from './sourceControlSetReadOnly';
import type { SourceControlGetStatus } from './sourceControlGetStatus';
export declare namespace SourceControlRequest {
type UpdatePreferences = AuthenticatedRequest<{}, {}, Partial<SourceControlPreferences>, {}>;
@@ -19,4 +20,5 @@ export declare namespace SourceControlRequest {
type Disconnect = AuthenticatedRequest<{}, {}, SourceControlDisconnect, {}>;
type PushWorkFolder = AuthenticatedRequest<{}, {}, SourceControlPushWorkFolder, {}>;
type PullWorkFolder = AuthenticatedRequest<{}, {}, SourceControlPullWorkFolder, {}>;
type GetStatus = AuthenticatedRequest<{}, {}, {}, SourceControlGetStatus>;
}

32
packages/cli/src/environments/sourceControl/types/sourceControlGetStatus.ts Normal file
View File

@@ -0,0 +1,32 @@
import { IsBoolean, IsOptional, IsString } from 'class-validator';
function booleanFromString(value: string | boolean): boolean {
if (typeof value === 'boolean') {
return value;
}
return value === 'true';
}
export class SourceControlGetStatus {
@IsString()
@IsOptional()
direction: 'push' | 'pull';
@IsBoolean()
@IsOptional()
preferLocalVersion: boolean;
@IsBoolean()
@IsOptional()
verbose: boolean;
constructor(values: {
direction: 'push' | 'pull';
preferLocalVersion: string | boolean;
verbose: string | boolean;
}) {
this.direction = values.direction || 'push';
this.preferLocalVersion = booleanFromString(values.preferLocalVersion) || true;
this.verbose = booleanFromString(values.verbose) || false;
}
}

10
packages/cli/src/environments/sourceControl/types/sourceControlPreferences.ts
View File

@@ -1,4 +1,4 @@
import { IsBoolean, IsEmail, IsHexColor, IsOptional, IsString } from 'class-validator';
import { IsBoolean, IsHexColor, IsOptional, IsString } from 'class-validator';
export class SourceControlPreferences {
constructor(preferences: Partial<SourceControlPreferences> | undefined = undefined) {
@@ -11,12 +11,6 @@ export class SourceControlPreferences {
@IsString()
repositoryUrl: string;
@IsString()
authorName: string;
@IsEmail()
authorEmail: string;
@IsString()
branchName = 'main';
@@ -45,8 +39,6 @@ export class SourceControlPreferences {
return new SourceControlPreferences({
connected: preferences.connected ?? defaultPreferences.connected,
repositoryUrl: preferences.repositoryUrl ?? defaultPreferences.repositoryUrl,
authorName: preferences.authorName ?? defaultPreferences.authorName,
authorEmail: preferences.authorEmail ?? defaultPreferences.authorEmail,
branchName: preferences.branchName ?? defaultPreferences.branchName,
branchReadOnly: preferences.branchReadOnly ?? defaultPreferences.branchReadOnly,
branchColor: preferences.branchColor ?? defaultPreferences.branchColor,

2
packages/cli/src/environments/sourceControl/types/sourceControlPullWorkFolder.ts
View File

@@ -24,6 +24,4 @@ export class SourceControllPullOptions {
force?: boolean;
variables?: { [key: string]: string };
importAfterPull?: boolean = true;
}

12
packages/cli/src/environments/sourceControl/types/sourceControlPushWorkFolder.ts
View File

@@ -1,4 +1,5 @@
import { IsBoolean, IsOptional, IsString } from 'class-validator';
import type { SourceControlledFile } from './sourceControlledFile';
export class SourceControlPushWorkFolder {
@IsBoolean()
@@ -6,16 +7,7 @@ export class SourceControlPushWorkFolder {
force?: boolean;
@IsString({ each: true })
@IsOptional()
fileNames?: Set<string>;
@IsString({ each: true })
@IsOptional()
workflowIds?: Set<string>;
@IsString({ each: true })
@IsOptional()
credentialIds?: Set<string>;
fileNames: SourceControlledFile[];
@IsString()
@IsOptional()

9
packages/cli/src/environments/sourceControl/types/sourceControlWorkflowVersionId.ts Normal file
View File

@@ -0,0 +1,9 @@
export interface SourceControlWorkflowVersionId {
id: string;
versionId: string;
filename: string;
name?: string;
localId?: string;
remoteId?: string;
updatedAt?: string;
}

3
packages/cli/src/environments/sourceControl/types/sourceControlledFile.ts
View File

@@ -5,6 +5,8 @@ export type SourceControlledFileStatus =
| 'created'
| 'renamed'
| 'conflicted'
| 'ignored'
| 'staged'
| 'unknown';
export type SourceControlledFileLocation = 'local' | 'remote';
export type SourceControlledFileType = 'credential' | 'workflow' | 'tags' | 'variables' | 'file';
@@ -17,4 +19,5 @@ export type SourceControlledFile = {
location: SourceControlledFileLocation;
conflict: boolean;
updatedAt: string;
pushed?: boolean;
};

2
packages/cli/src/environments/variables/variables.service.ee.ts
View File

@@ -21,7 +21,7 @@ export class EEVariablesService extends VariablesService {
if (variable.key.replace(/[A-Za-z0-9_]/g, '').length !== 0) {
throw new VariablesValidationError('key can only contain characters A-Za-z0-9_');
}
if (variable.value.length > 255) {
if (variable.value?.length > 255) {
throw new VariablesValidationError('value cannot be longer than 255 characters');
}
}

10
packages/cli/src/telemetry/index.ts
View File

@@ -10,7 +10,8 @@ import { getLogger } from '@/Logger';
import { License } from '@/License';
import { LicenseService } from '@/license/License.service';
import { N8N_VERSION } from '@/constants';
import { Service } from 'typedi';
import Container, { Service } from 'typedi';
import { SourceControlPreferencesService } from '../environments/sourceControl/sourceControlPreferences.service.ee';
type ExecutionTrackDataKey = 'manual_error' | 'manual_success' | 'prod_error' | 'prod_success';
@@ -105,11 +106,18 @@ export class Telemetry {
this.executionCountsBuffer = {};
const sourceControlPreferences = Container.get(
SourceControlPreferencesService,
).getPreferences();
// License info
const pulsePacket = {
plan_name_current: this.license.getPlanName(),
quota: this.license.getTriggerLimit(),
usage: await LicenseService.getActiveTriggerCount(),
source_control_set_up: Container.get(SourceControlPreferencesService).isSourceControlSetup(),
branchName: sourceControlPreferences.branchName,
read_only_instance: sourceControlPreferences.branchReadOnly,
};
allPromises.push(this.track('pulse', pulsePacket));
return Promise.all(allPromises);

28
packages/cli/src/workflows/workflows.services.ts
View File

@@ -222,16 +222,24 @@ export class WorkflowsService {
);
}
// Update the workflow's version
workflow.versionId = uuid();
LoggerProxy.verbose(
`Updating versionId for workflow ${workflowId} for user ${user.id} after saving`,
{
previousVersionId: shared.workflow.versionId,
newVersionId: workflow.versionId,
},
);
if (
Object.keys(workflow).length === 3 &&
workflow.id !== undefined &&
workflow.versionId !== undefined &&
workflow.active !== undefined
) {
// we're just updating the active status of the workflow, don't update the versionId
} else {
// Update the workflow's version
workflow.versionId = uuid();
LoggerProxy.verbose(
`Updating versionId for workflow ${workflowId} for user ${user.id} after saving`,
{
previousVersionId: shared.workflow.versionId,
newVersionId: workflow.versionId,
},
);
}
// check credentials for old format
await WorkflowHelpers.replaceInvalidCredentials(workflow);