feat: Environments release using source control (#6653)
* initial telemetry setup and adjusted pull return * quicksave before merge * feat: add conflicting workflow list to pull modal * feat: update source control pull modal * fix: fix linting issue * feat: add Enter keydown event for submitting source control push modal (no-changelog) feat: add Enter keydown event for submitting source control push modal * quicksave * user workflow table for export * improve telemetry data * pull api telemetry * fix lint * Copy tweaks. * remove authorName and authorEmail and pick from user * rename owners.json to workflow_owners.json * ignore credential conflicts on pull * feat: several push/pull flow changes and design update * pull and push return same data format * fix: add One last step toast for successful pull * feat: add up to date pull toast * fix: add proper Learn more link for push and pull modals * do not await tracking being sent * fix import * fix await * add more sourcecontrolfile status * Minor copy tweak for "More info". * Minor copy tweak for "More info". * ignore variable_stub conflicts on pull * ignore whitespace differences * do not show remote workflows that are not yet created * fix telemetry * fix toast when pulling deleted wf * lint fix * refactor and make some imports dynamic * fix variable edit validation * fix telemetry response * improve telemetry * fix unintenional delete commit * fix status unknown issue * fix up to date toast * do not export active state and reapply versionid * use update instead of upsert * fix: show all workflows when clicking push to git * feat: update Up to date pull translation * fix: update read only env checks * do not update versionid of only active flag changes * feat: prevent access to new workflow and templates import when read only env * feat: send only active state and version if workflow state is not dirty * fix: Detect when only active state has changed and prevent generation a new version ID * feat: improve readonly env messages * make getPreferences public * fix telemetry issue * fix: add partial workflow update based on dirty state when changing active state * update unit tests * fix: remove unsaved changes check in readOnlyEnv * fix: disable push to git button when read onyl env * fix: update readonly toast duration * fix: fix pinning and title input in protected mode * initial commit (NOT working) * working push * cleanup and implement pull * fix getstatus * update import to new method * var and tag diffs are no conflicts * only show pull conflict for workflows * refactor and ignore faulty credentials * add sanitycheck for missing git folder * prefer fetch over pull and limit depth to 1 * back to pull... * fix setting branch on initial connect * fix test * remove clean workfolder * refactor: Remove some unnecessary code * Fixed links to docs. * fix getstatus query params * lint fix * dialog to show local and remote name on conflict * only show remote name on conflict * fix credential expression export * fix: Broken test * dont show toast on pull with empty var/tags and refactor * apply frontend changes from old branch * fix tag with same name import * fix buttons shown for non instance owners * prepare local storage key for removal * refactor: Change wording on pushing and pulling * refactor: Change menu item * test: Fix broken test * Update packages/cli/src/environments/sourceControl/types/sourceControlPushWorkFolder.ts Co-authored-by: Iván Ovejero <ivov.src@gmail.com> --------- Co-authored-by: Alex Grozav <alex@grozav.com> Co-authored-by: Giulio Andreini <g.andreini@gmail.com> Co-authored-by: Omar Ajoue <krynble@gmail.com> Co-authored-by: Iván Ovejero <ivov.src@gmail.com>
This commit is contained in:
Michael Auerswald
committed by
GitHub
GitHub
parent
bcfc5e717b
commit
fc7aa8bd66
@@ -3,23 +3,28 @@ import path from 'path';
|
||||
import {
|
||||
SOURCE_CONTROL_CREDENTIAL_EXPORT_FOLDER,
|
||||
SOURCE_CONTROL_GIT_FOLDER,
|
||||
SOURCE_CONTROL_OWNERS_EXPORT_FILE,
|
||||
SOURCE_CONTROL_TAGS_EXPORT_FILE,
|
||||
SOURCE_CONTROL_VARIABLES_EXPORT_FILE,
|
||||
SOURCE_CONTROL_WORKFLOW_EXPORT_FOLDER,
|
||||
} from './constants';
|
||||
import * as Db from '@/Db';
|
||||
import glob from 'fast-glob';
|
||||
import type { ICredentialDataDecryptedObject } from 'n8n-workflow';
|
||||
import { LoggerProxy, jsonParse } from 'n8n-workflow';
|
||||
import { writeFile as fsWriteFile, readFile as fsReadFile, rm as fsRm } from 'fs/promises';
|
||||
import { LoggerProxy } from 'n8n-workflow';
|
||||
import { writeFile as fsWriteFile, rm as fsRm } from 'fs/promises';
|
||||
import { rmSync } from 'fs';
|
||||
import { Credentials, UserSettings } from 'n8n-core';
|
||||
import type { IWorkflowToImport } from '@/Interfaces';
|
||||
import type { ExportableWorkflow } from './types/exportableWorkflow';
|
||||
import type { ExportableCredential } from './types/exportableCredential';
|
||||
import type { ExportResult } from './types/exportResult';
|
||||
import type { SharedWorkflow } from '@db/entities/SharedWorkflow';
|
||||
import { sourceControlFoldersExistCheck } from './sourceControlHelper.ee';
|
||||
import {
|
||||
getCredentialExportPath,
|
||||
getVariablesPath,
|
||||
getWorkflowExportPath,
|
||||
sourceControlFoldersExistCheck,
|
||||
stringContainsExpression,
|
||||
} from './sourceControlHelper.ee';
|
||||
import type { WorkflowEntity } from '@db/entities/WorkflowEntity';
|
||||
import { In } from 'typeorm';
|
||||
import type { SourceControlledFile } from './types/sourceControlledFile';
|
||||
|
||||
@Service()
|
||||
export class SourceControlExportService {
|
||||
@@ -40,79 +45,11 @@ export class SourceControlExportService {
|
||||
}
|
||||
|
||||
getWorkflowPath(workflowId: string): string {
|
||||
return path.join(this.workflowExportFolder, `${workflowId}.json`);
|
||||
return getWorkflowExportPath(workflowId, this.workflowExportFolder);
|
||||
}
|
||||
|
||||
getCredentialsPath(credentialsId: string): string {
|
||||
return path.join(this.credentialExportFolder, `${credentialsId}.json`);
|
||||
}
|
||||
|
||||
getTagsPath(): string {
|
||||
return path.join(this.gitFolder, SOURCE_CONTROL_TAGS_EXPORT_FILE);
|
||||
}
|
||||
|
||||
getOwnersPath(): string {
|
||||
return path.join(this.gitFolder, SOURCE_CONTROL_OWNERS_EXPORT_FILE);
|
||||
}
|
||||
|
||||
getVariablesPath(): string {
|
||||
return path.join(this.gitFolder, SOURCE_CONTROL_VARIABLES_EXPORT_FILE);
|
||||
}
|
||||
|
||||
async getWorkflowFromFile(
|
||||
filePath: string,
|
||||
root = this.gitFolder,
|
||||
): Promise<IWorkflowToImport | undefined> {
|
||||
try {
|
||||
const importedWorkflow = jsonParse<IWorkflowToImport>(
|
||||
await fsReadFile(path.join(root, filePath), { encoding: 'utf8' }),
|
||||
);
|
||||
return importedWorkflow;
|
||||
} catch (error) {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async getCredentialFromFile(
|
||||
filePath: string,
|
||||
root = this.gitFolder,
|
||||
): Promise<ExportableCredential | undefined> {
|
||||
try {
|
||||
const credential = jsonParse<ExportableCredential>(
|
||||
await fsReadFile(path.join(root, filePath), { encoding: 'utf8' }),
|
||||
);
|
||||
return credential;
|
||||
} catch (error) {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async cleanWorkFolder() {
|
||||
try {
|
||||
const workflowFiles = await glob('*.json', {
|
||||
cwd: this.workflowExportFolder,
|
||||
absolute: true,
|
||||
});
|
||||
const credentialFiles = await glob('*.json', {
|
||||
cwd: this.credentialExportFolder,
|
||||
absolute: true,
|
||||
});
|
||||
const variablesFile = await glob(SOURCE_CONTROL_VARIABLES_EXPORT_FILE, {
|
||||
cwd: this.gitFolder,
|
||||
absolute: true,
|
||||
});
|
||||
const tagsFile = await glob(SOURCE_CONTROL_TAGS_EXPORT_FILE, {
|
||||
cwd: this.gitFolder,
|
||||
absolute: true,
|
||||
});
|
||||
await Promise.all(tagsFile.map(async (e) => fsRm(e)));
|
||||
await Promise.all(variablesFile.map(async (e) => fsRm(e)));
|
||||
await Promise.all(workflowFiles.map(async (e) => fsRm(e)));
|
||||
await Promise.all(credentialFiles.map(async (e) => fsRm(e)));
|
||||
LoggerProxy.debug('Cleaned work folder.');
|
||||
} catch (error) {
|
||||
LoggerProxy.error(`Failed to clean work folder: ${(error as Error).message}`);
|
||||
}
|
||||
return getCredentialExportPath(credentialsId, this.credentialExportFolder);
|
||||
}
|
||||
|
||||
async deleteRepositoryFolder() {
|
||||
@@ -123,86 +60,73 @@ export class SourceControlExportService {
|
||||
}
|
||||
}
|
||||
|
||||
private async rmDeletedWorkflowsFromExportFolder(
|
||||
workflowsToBeExported: SharedWorkflow[],
|
||||
): Promise<Set<string>> {
|
||||
const sharedWorkflowsFileNames = new Set<string>(
|
||||
workflowsToBeExported.map((e) => this.getWorkflowPath(e?.workflow?.name)),
|
||||
);
|
||||
const existingWorkflowsInFolder = new Set<string>(
|
||||
await glob('*.json', {
|
||||
cwd: this.workflowExportFolder,
|
||||
absolute: true,
|
||||
}),
|
||||
);
|
||||
const deletedWorkflows = new Set(existingWorkflowsInFolder);
|
||||
for (const elem of sharedWorkflowsFileNames) {
|
||||
deletedWorkflows.delete(elem);
|
||||
}
|
||||
public rmFilesFromExportFolder(filesToBeDeleted: Set<string>): Set<string> {
|
||||
try {
|
||||
await Promise.all([...deletedWorkflows].map(async (e) => fsRm(e)));
|
||||
filesToBeDeleted.forEach((e) => rmSync(e));
|
||||
} catch (error) {
|
||||
LoggerProxy.error(`Failed to delete workflows from work folder: ${(error as Error).message}`);
|
||||
}
|
||||
return deletedWorkflows;
|
||||
return filesToBeDeleted;
|
||||
}
|
||||
|
||||
private async writeExportableWorkflowsToExportFolder(workflowsToBeExported: SharedWorkflow[]) {
|
||||
private async writeExportableWorkflowsToExportFolder(
|
||||
workflowsToBeExported: WorkflowEntity[],
|
||||
owners: Record<string, string>,
|
||||
) {
|
||||
await Promise.all(
|
||||
workflowsToBeExported.map(async (e) => {
|
||||
if (!e.workflow) {
|
||||
LoggerProxy.debug(
|
||||
`Found no corresponding workflow ${e.workflowId ?? 'unknown'}, skipping export`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const fileName = this.getWorkflowPath(e.workflow?.id);
|
||||
const fileName = this.getWorkflowPath(e.id);
|
||||
const sanitizedWorkflow: ExportableWorkflow = {
|
||||
active: e.workflow?.active,
|
||||
id: e.workflow?.id,
|
||||
name: e.workflow?.name,
|
||||
nodes: e.workflow?.nodes,
|
||||
connections: e.workflow?.connections,
|
||||
settings: e.workflow?.settings,
|
||||
triggerCount: e.workflow?.triggerCount,
|
||||
versionId: e.workflow?.versionId,
|
||||
id: e.id,
|
||||
name: e.name,
|
||||
nodes: e.nodes,
|
||||
connections: e.connections,
|
||||
settings: e.settings,
|
||||
triggerCount: e.triggerCount,
|
||||
versionId: e.versionId,
|
||||
owner: owners[e.id],
|
||||
};
|
||||
LoggerProxy.debug(`Writing workflow ${e.workflowId} to ${fileName}`);
|
||||
LoggerProxy.debug(`Writing workflow ${e.id} to ${fileName}`);
|
||||
return fsWriteFile(fileName, JSON.stringify(sanitizedWorkflow, null, 2));
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async exportWorkflowsToWorkFolder(): Promise<ExportResult> {
|
||||
async exportWorkflowsToWorkFolder(candidates: SourceControlledFile[]): Promise<ExportResult> {
|
||||
try {
|
||||
sourceControlFoldersExistCheck([this.workflowExportFolder]);
|
||||
const workflowIds = candidates.map((e) => e.id);
|
||||
const sharedWorkflows = await Db.collections.SharedWorkflow.find({
|
||||
relations: ['workflow', 'role', 'user'],
|
||||
relations: ['role', 'user'],
|
||||
where: {
|
||||
role: {
|
||||
name: 'owner',
|
||||
scope: 'workflow',
|
||||
},
|
||||
workflowId: In(workflowIds),
|
||||
},
|
||||
});
|
||||
const workflows = await Db.collections.Workflow.find({
|
||||
where: {
|
||||
id: In(workflowIds),
|
||||
},
|
||||
});
|
||||
|
||||
// before exporting, figure out which workflows have been deleted and remove them from the export folder
|
||||
const removedFiles = await this.rmDeletedWorkflowsFromExportFolder(sharedWorkflows);
|
||||
// write the workflows to the export folder as json files
|
||||
await this.writeExportableWorkflowsToExportFolder(sharedWorkflows);
|
||||
// write list of owners to file
|
||||
const ownersFileName = this.getOwnersPath();
|
||||
// determine owner of each workflow to be exported
|
||||
const owners: Record<string, string> = {};
|
||||
sharedWorkflows.forEach((e) => (owners[e.workflowId] = e.user.email));
|
||||
await fsWriteFile(ownersFileName, JSON.stringify(owners, null, 2));
|
||||
|
||||
// write the workflows to the export folder as json files
|
||||
await this.writeExportableWorkflowsToExportFolder(workflows, owners);
|
||||
|
||||
// await fsWriteFile(ownersFileName, JSON.stringify(owners, null, 2));
|
||||
return {
|
||||
count: sharedWorkflows.length,
|
||||
folder: this.workflowExportFolder,
|
||||
files: sharedWorkflows.map((e) => ({
|
||||
id: e?.workflow?.id,
|
||||
name: this.getWorkflowPath(e?.workflow?.name),
|
||||
files: workflows.map((e) => ({
|
||||
id: e?.id,
|
||||
name: this.getWorkflowPath(e?.name),
|
||||
})),
|
||||
removedFiles: [...removedFiles],
|
||||
};
|
||||
} catch (error) {
|
||||
throw Error(`Failed to export workflows to work folder: ${(error as Error).message}`);
|
||||
@@ -221,7 +145,7 @@ export class SourceControlExportService {
|
||||
files: [],
|
||||
};
|
||||
}
|
||||
const fileName = this.getVariablesPath();
|
||||
const fileName = getVariablesPath(this.gitFolder);
|
||||
const sanitizedVariables = variables.map((e) => ({ ...e, value: '' }));
|
||||
await fsWriteFile(fileName, JSON.stringify(sanitizedVariables, null, 2));
|
||||
return {
|
||||
@@ -252,7 +176,7 @@ export class SourceControlExportService {
|
||||
};
|
||||
}
|
||||
const mappings = await Db.collections.WorkflowTagMapping.find();
|
||||
const fileName = this.getTagsPath();
|
||||
const fileName = path.join(this.gitFolder, SOURCE_CONTROL_TAGS_EXPORT_FILE);
|
||||
await fsWriteFile(
|
||||
fileName,
|
||||
JSON.stringify(
|
||||
@@ -289,10 +213,7 @@ export class SourceControlExportService {
|
||||
} else if (typeof data[key] === 'object') {
|
||||
data[key] = this.replaceCredentialData(data[key] as ICredentialDataDecryptedObject);
|
||||
} else if (typeof data[key] === 'string') {
|
||||
data[key] =
|
||||
(data[key] as string)?.startsWith('={{') && (data[key] as string)?.includes('$secret')
|
||||
? data[key]
|
||||
: '';
|
||||
data[key] = stringContainsExpression(data[key] as string) ? data[key] : '';
|
||||
} else if (typeof data[key] === 'number') {
|
||||
// TODO: leaving numbers in for now, but maybe we should remove them
|
||||
continue;
|
||||
@@ -305,23 +226,31 @@ export class SourceControlExportService {
|
||||
return data;
|
||||
};
|
||||
|
||||
async exportCredentialsToWorkFolder(): Promise<ExportResult> {
|
||||
async exportCredentialsToWorkFolder(candidates: SourceControlledFile[]): Promise<ExportResult> {
|
||||
try {
|
||||
sourceControlFoldersExistCheck([this.credentialExportFolder]);
|
||||
const sharedCredentials = await Db.collections.SharedCredentials.find({
|
||||
const credentialIds = candidates.map((e) => e.id);
|
||||
const credentialsToBeExported = await Db.collections.SharedCredentials.find({
|
||||
relations: ['credentials', 'role', 'user'],
|
||||
where: {
|
||||
credentialsId: In(credentialIds),
|
||||
},
|
||||
});
|
||||
let missingIds: string[] = [];
|
||||
if (credentialsToBeExported.length !== credentialIds.length) {
|
||||
const foundCredentialIds = credentialsToBeExported.map((e) => e.credentialsId);
|
||||
missingIds = credentialIds.filter(
|
||||
(remote) => foundCredentialIds.findIndex((local) => local === remote) === -1,
|
||||
);
|
||||
}
|
||||
const encryptionKey = await UserSettings.getEncryptionKey();
|
||||
await Promise.all(
|
||||
sharedCredentials.map(async (sharedCredential) => {
|
||||
credentialsToBeExported.map(async (sharedCredential) => {
|
||||
const { name, type, nodesAccess, data, id } = sharedCredential.credentials;
|
||||
const credentialObject = new Credentials({ id, name }, type, nodesAccess, data);
|
||||
const plainData = credentialObject.getData(encryptionKey);
|
||||
const sanitizedData = this.replaceCredentialData(plainData);
|
||||
const fileName = path.join(
|
||||
this.credentialExportFolder,
|
||||
`${sharedCredential.credentials.id}.json`,
|
||||
);
|
||||
const fileName = this.getCredentialsPath(sharedCredential.credentials.id);
|
||||
const sanitizedCredential: ExportableCredential = {
|
||||
id: sharedCredential.credentials.id,
|
||||
name: sharedCredential.credentials.name,
|
||||
@@ -334,12 +263,13 @@ export class SourceControlExportService {
|
||||
}),
|
||||
);
|
||||
return {
|
||||
count: sharedCredentials.length,
|
||||
count: credentialsToBeExported.length,
|
||||
folder: this.credentialExportFolder,
|
||||
files: sharedCredentials.map((e) => ({
|
||||
files: credentialsToBeExported.map((e) => ({
|
||||
id: e.credentials.id,
|
||||
name: path.join(this.credentialExportFolder, `${e.credentials.name}.json`),
|
||||
})),
|
||||
missingIds,
|
||||
};
|
||||
} catch (error) {
|
||||
throw Error(`Failed to export credentials to work folder: ${(error as Error).message}`);
|
||||
|
||||
Reference in New Issue
Block a user