mohiit1502/Automata
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ci(core): Reduce memory usage in tests (part-2) (no-changelog) (#7671)

Browse Source 
This also gets rid of `Db.collection`, which was another source of
circular dependencies.
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™
2023-11-10 15:04:26 +01:00
committed by GitHub
parent 37dd658dc5
commit 000e76e3b4
174 changed files with 892 additions and 808 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
packages/cli/src/environments/sourceControl/sourceControlImport.service.ee.ts
View File

@@ -7,7 +7,6 @@ import {
SOURCE_CONTROL_VARIABLES_EXPORT_FILE,
SOURCE_CONTROL_WORKFLOW_EXPORT_FOLDER,
} from './constants';
import * as Db from '@/Db';
import glob from 'fast-glob';
import { jsonParse } from 'n8n-workflow';
import { readFile as fsReadFile } from 'fs/promises';
@@ -26,9 +25,16 @@ import { getCredentialExportPath, getWorkflowExportPath } from './sourceControlH
import type { SourceControlledFile } from './types/sourceControlledFile';
import { RoleService } from '@/services/role.service';
import { VariablesService } from '../variables/variables.service';
import { TagRepository } from '@/databases/repositories';
import { TagRepository } from '@db/repositories/tag.repository';
import { WorkflowRepository } from '@db/repositories/workflow.repository';
import { UserRepository } from '@db/repositories/user.repository';
import { UM_FIX_INSTRUCTION } from '@/constants';
import { Logger } from '@/Logger';
import { CredentialsRepository } from '@db/repositories/credentials.repository';
import { SharedCredentialsRepository } from '@db/repositories/sharedCredentials.repository';
import { SharedWorkflowRepository } from '@db/repositories/sharedWorkflow.repository';
import { WorkflowTagMappingRepository } from '@db/repositories/workflowTagMapping.repository';
import { VariablesRepository } from '@db/repositories/variables.repository';
@Service()
export class SourceControlImportService {
@@ -110,7 +116,7 @@ export class SourceControlImportService {
}
public async getLocalVersionIdsFromDb(): Promise<SourceControlWorkflowVersionId[]> {
const localWorkflows = await Db.collections.Workflow.find({
const localWorkflows = await Container.get(WorkflowRepository).find({
select: ['id', 'name', 'versionId', 'updatedAt'],
});
return localWorkflows.map((local) => ({
@@ -153,7 +159,7 @@ export class SourceControlImportService {
public async getLocalCredentialsFromDb(): Promise<
Array<ExportableCredential & { filename: string }>
> {
const localCredentials = await Db.collections.Credentials.find({
const localCredentials = await Container.get(CredentialsRepository).find({
select: ['id', 'name', 'type', 'nodesAccess'],
});
return localCredentials.map((local) => ({
@@ -209,7 +215,7 @@ export class SourceControlImportService {
const localTags = await this.tagRepository.find({
select: ['id', 'name'],
});
const localMappings = await Db.collections.WorkflowTagMapping.find({
const localMappings = await Container.get(WorkflowTagMappingRepository).find({
select: ['workflowId', 'tagId'],
});
return { tags: localTags, mappings: localMappings };
@@ -219,13 +225,13 @@ export class SourceControlImportService {
const ownerWorkflowRole = await this.getWorkflowOwnerRole();
const workflowRunner = this.activeWorkflowRunner;
const candidateIds = candidates.map((c) => c.id);
const existingWorkflows = await Db.collections.Workflow.find({
const existingWorkflows = await Container.get(WorkflowRepository).find({
where: {
id: In(candidateIds),
},
select: ['id', 'name', 'versionId', 'active'],
});
const allSharedWorkflows = await Db.collections.SharedWorkflow.find({
const allSharedWorkflows = await Container.get(SharedWorkflowRepository).find({
where: {
workflowId: In(candidateIds),
},
@@ -244,7 +250,10 @@ export class SourceControlImportService {
const existingWorkflow = existingWorkflows.find((e) => e.id === importedWorkflow.id);
importedWorkflow.active = existingWorkflow?.active ?? false;
this.logger.debug(`Updating workflow id ${importedWorkflow.id ?? 'new'}`);
const upsertResult = await Db.collections.Workflow.upsert({ ...importedWorkflow }, ['id']);
const upsertResult = await Container.get(WorkflowRepository).upsert(
{ ...importedWorkflow },
['id'],
);
if (upsertResult?.identifiers?.length !== 1) {
throw new Error(`Failed to upsert workflow ${importedWorkflow.id ?? 'new'}`);
}
@@ -254,7 +263,7 @@ export class SourceControlImportService {
if (cachedOwnerIds.has(importedWorkflow.owner)) {
workflowOwnerId = cachedOwnerIds.get(importedWorkflow.owner) ?? userId;
} else {
const foundUser = await Db.collections.User.findOne({
const foundUser = await Container.get(UserRepository).findOne({
where: {
email: importedWorkflow.owner,
},
@@ -278,7 +287,7 @@ export class SourceControlImportService {
);
if (!existingSharedWorkflowOwnerByUserId && !existingSharedWorkflowOwnerByRoleId) {
// no owner exists yet, so create one
await Db.collections.SharedWorkflow.insert({
await Container.get(SharedWorkflowRepository).insert({
workflowId: importedWorkflow.id,
userId: workflowOwnerId,
roleId: ownerWorkflowRole.id,
@@ -288,7 +297,7 @@ export class SourceControlImportService {
} else if (existingSharedWorkflowOwnerByUserId && !existingSharedWorkflowOwnerByRoleId) {
// if the worklflow has a non-global owner that is referenced by the owner file,
// and no existing global owner, update the owner to the user referenced in the owner file
await Db.collections.SharedWorkflow.update(
await Container.get(SharedWorkflowRepository).update(
{
workflowId: importedWorkflow.id,
userId: workflowOwnerId,
@@ -310,7 +319,7 @@ export class SourceControlImportService {
} catch (error) {
this.logger.error(`Failed to activate workflow ${existingWorkflow.id}`, error as Error);
} finally {
await Db.collections.Workflow.update(
await Container.get(WorkflowRepository).update(
{ id: existingWorkflow.id },
{ versionId: importedWorkflow.versionId },
);
@@ -331,7 +340,7 @@ export class SourceControlImportService {
public async importCredentialsFromWorkFolder(candidates: SourceControlledFile[], userId: string) {
const candidateIds = candidates.map((c) => c.id);
const existingCredentials = await Db.collections.Credentials.find({
const existingCredentials = await Container.get(CredentialsRepository).find({
where: {
id: In(candidateIds),
},
@@ -339,7 +348,7 @@ export class SourceControlImportService {
});
const ownerCredentialRole = await this.getCredentialOwnerRole();
const ownerGlobalRole = await this.getOwnerGlobalRole();
const existingSharedCredentials = await Db.collections.SharedCredentials.find({
const existingSharedCredentials = await Container.get(SharedCredentialsRepository).find({
select: ['userId', 'credentialsId', 'roleId'],
where: {
credentialsId: In(candidateIds),
@@ -370,7 +379,7 @@ export class SourceControlImportService {
newCredentialObject.nodesAccess = nodesAccess || existingCredential?.nodesAccess || [];
this.logger.debug(`Updating credential id ${newCredentialObject.id as string}`);
await Db.collections.Credentials.upsert(newCredentialObject, ['id']);
await Container.get(CredentialsRepository).upsert(newCredentialObject, ['id']);
if (!sharedOwner) {
const newSharedCredential = new SharedCredentials();
@@ -378,7 +387,7 @@ export class SourceControlImportService {
newSharedCredential.userId = userId;
newSharedCredential.roleId = ownerCredentialRole.id;
await Db.collections.SharedCredentials.upsert({ ...newSharedCredential }, [
await Container.get(SharedCredentialsRepository).upsert({ ...newSharedCredential }, [
'credentialsId',
'userId',
]);
@@ -413,7 +422,7 @@ export class SourceControlImportService {
const existingWorkflowIds = new Set(
(
await Db.collections.Workflow.find({
await Container.get(WorkflowRepository).find({
select: ['id'],
})
).map((e) => e.id),
@@ -442,7 +451,7 @@ export class SourceControlImportService {
await Promise.all(
mappedTags.mappings.map(async (mapping) => {
if (!existingWorkflowIds.has(String(mapping.workflowId))) return;
await Db.collections.WorkflowTagMapping.upsert(
await Container.get(WorkflowTagMappingRepository).upsert(
{ tagId: String(mapping.tagId), workflowId: String(mapping.workflowId) },
{
skipUpdateIfNoValuesChanged: true,
@@ -489,12 +498,12 @@ export class SourceControlImportService {
overriddenKeys.splice(overriddenKeys.indexOf(variable.key), 1);
}
try {
await Db.collections.Variables.upsert({ ...variable }, ['id']);
await Container.get(VariablesRepository).upsert({ ...variable }, ['id']);
} catch (errorUpsert) {
if (isUniqueConstraintError(errorUpsert as Error)) {
this.logger.debug(`Variable ${variable.key} already exists, updating instead`);
try {
await Db.collections.Variables.update({ key: variable.key }, { ...variable });
await Container.get(VariablesRepository).update({ key: variable.key }, { ...variable });
} catch (errorUpdate) {
this.logger.debug(`Failed to update variable ${variable.key}, skipping`);
this.logger.debug((errorUpdate as Error).message);
@@ -509,8 +518,11 @@ export class SourceControlImportService {
if (overriddenKeys.length > 0 && valueOverrides) {
for (const key of overriddenKeys) {
result.imported.push(key);
const newVariable = Db.collections.Variables.create({ key, value: valueOverrides[key] });
await Db.collections.Variables.save(newVariable);
const newVariable = Container.get(VariablesRepository).create({
key,
value: valueOverrides[key],
});
await Container.get(VariablesRepository).save(newVariable);
}
}