perf(core): Add filtering and pagination to GET /workflows (#6845)

* Initial setup

* Specify max paginated items

* Simplify

* Add tests

* Add more tests

* Add migrations

* Add top-level property

* Add field selection

* Cleanup

* Rename `total` to `count`

* More cleanup

* Move query logic into `WorkflowRepository`

* Create `AbstractRepository`

* Cleanup

* Fix name

* Remove leftover comments

* Replace reference

* Add default for `rawSkip`

* Remove unneeded typing

* Switch to `class-validator`

* Simplify

* Simplify

* Type as optional

* Make typing more accurate

* Fix lint

* Use `getOwnPropertyNames`

* Use DSL

* Set schema at repo level

* Cleanup

* Remove comment

* Refactor repository methods to middleware

* Add middleware tests

* Remove old test files

* Remove generic experiment

* Reuse `reportError`

* Remove unused type

* Cleanup

* Improve wording

* Reduce diff

* Add missing mw

* Use `Container.get`

* Adjust lint rule

* Reorganize into subdir

* Remove unused directive

* Remove nodes

* Silly mistake

* Validate take

* refactor(core): Adjust index handling in new migrations DSL (no-changelog) (#6876)

* refactor(core): Adjust index handling in new migrations DSL (no-changelog)

* Account for custom index name

* Also for dropping

* Fix `select` issue with `relations`

* Tighten validation

* Ensure `ownerId` is not added when specifying `select`

packages/cli/src/workflows/workflows.controller.ee.ts

@@ -6,7 +6,7 @@ import * as WorkflowHelpers from '@/WorkflowHelpers';
 import config from '@/config';
 import { WorkflowEntity } from '@db/entities/WorkflowEntity';
 import { validateEntity } from '@/GenericHelpers';
-import type { WorkflowRequest } from '@/requests';
+import type { ListQueryRequest, WorkflowRequest } from '@/requests';
 import { isSharingEnabled, rightDiff } from '@/UserManagement/UserManagementHelper';
 import { EEWorkflowsService as EEWorkflows } from './workflows.services.ee';
 import { ExternalHooks } from '@/ExternalHooks';
@@ -20,6 +20,8 @@ import { In } from 'typeorm';
 import { Container } from 'typedi';
 import { InternalHooks } from '@/InternalHooks';
 import { RoleService } from '@/services/role.service';
+import * as utils from '@/utils';
+import { listQueryMiddleware } from '@/middlewares';
 import { TagRepository } from '@/databases/repositories';
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
@@ -203,17 +205,27 @@ EEWorkflowController.post(
  */
 EEWorkflowController.get(
 	'/',
-	ResponseHelper.send(async (req: WorkflowRequest.GetAll) => {
-		const [workflows, workflowOwnerRole] = await Promise.all([
-			EEWorkflows.getMany(req.user, req.query.filter),
-			Container.get(RoleService).findWorkflowOwnerRole(),
-		]);
+	listQueryMiddleware,
+	async (req: ListQueryRequest, res: express.Response) => {
+		try {
+			const [workflows, count] = await EEWorkflows.getMany(req.user, req.listQueryOptions);
 
-		return workflows.map((workflow) => {
-			EEWorkflows.addOwnerId(workflow, workflowOwnerRole);
-			return workflow;
-		});
-	}),
+			let data;
+
+			if (req.listQueryOptions?.select) {
+				data = workflows;
+			} else {
+				const role = await Container.get(RoleService).findWorkflowOwnerRole();
+				data = workflows.map((w) => EEWorkflows.addOwnerId(w, role));
+			}
+
+			res.json({ count, data });
+		} catch (maybeError) {
+			const error = utils.toError(maybeError);
+			ResponseHelper.reportError(error);
+			ResponseHelper.sendErrorResponse(res, error);
+		}
+	},
 );
 
 EEWorkflowController.patch(

packages/cli/src/workflows/workflows.controller.ts

@@ -15,7 +15,7 @@ import { WorkflowEntity } from '@db/entities/WorkflowEntity';
 import { validateEntity } from '@/GenericHelpers';
 import { ExternalHooks } from '@/ExternalHooks';
 import { getLogger } from '@/Logger';
-import type { WorkflowRequest } from '@/requests';
+import type { ListQueryRequest, WorkflowRequest } from '@/requests';
 import { isBelowOnboardingThreshold } from '@/WorkflowHelpers';
 import { EEWorkflowController } from './workflows.controller.ee';
 import { WorkflowsService } from './workflows.services';
@@ -24,6 +24,8 @@ import { In } from 'typeorm';
 import { Container } from 'typedi';
 import { InternalHooks } from '@/InternalHooks';
 import { RoleService } from '@/services/role.service';
+import * as utils from '@/utils';
+import { listQueryMiddleware } from '@/middlewares';
 import { TagRepository } from '@/databases/repositories';
 
 export const workflowsController = express.Router();
@@ -116,9 +118,18 @@ workflowsController.post(
  */
 workflowsController.get(
 	'/',
-	ResponseHelper.send(async (req: WorkflowRequest.GetAll) => {
-		return WorkflowsService.getMany(req.user, req.query.filter);
-	}),
+	listQueryMiddleware,
+	async (req: ListQueryRequest, res: express.Response) => {
+		try {
+			const [data, count] = await WorkflowsService.getMany(req.user, req.listQueryOptions);
+
+			res.json({ count, data });
+		} catch (maybeError) {
+			const error = utils.toError(maybeError);
+			ResponseHelper.reportError(error);
+			ResponseHelper.sendErrorResponse(res, error);
+		}
+	},
 );
 
 /**

packages/cli/src/workflows/workflows.services.ee.ts

@@ -88,11 +88,12 @@ export class EEWorkflowsService extends WorkflowsService {
 		return transaction.save(newSharedWorkflows);
 	}
 
-	static addOwnerId(workflow: WorkflowForList, workflowOwnerRole: Role): void {
+	static addOwnerId(workflow: WorkflowForList, workflowOwnerRole: Role) {
 		const ownerId = workflow.shared?.find(({ roleId }) => String(roleId) === workflowOwnerRole.id)
 			?.userId;
 		workflow.ownedBy = ownerId ? { id: ownerId } : null;
 		delete workflow.shared;
+		return workflow;
 	}
 
 	static addOwnerAndSharings(workflow: WorkflowWithSharingsAndCredentials): void {

packages/cli/src/workflows/workflows.services.ts

@@ -1,9 +1,8 @@
 import { Container } from 'typedi';
-import { validate as jsonSchemaValidate } from 'jsonschema';
-import type { INode, IPinData, JsonObject } from 'n8n-workflow';
-import { NodeApiError, jsonParse, LoggerProxy, Workflow } from 'n8n-workflow';
-import type { FindOptionsSelect, FindOptionsWhere, UpdateResult } from 'typeorm';
-import { In } from 'typeorm';
+import type { INode, IPinData } from 'n8n-workflow';
+import { NodeApiError, LoggerProxy, Workflow } from 'n8n-workflow';
+import type { FindManyOptions, FindOptionsSelect, FindOptionsWhere, UpdateResult } from 'typeorm';
+import { In, Like } from 'typeorm';
 import pick from 'lodash/pick';
 import { v4 as uuid } from 'uuid';
 import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
@@ -18,7 +17,7 @@ import type { WorkflowEntity } from '@db/entities/WorkflowEntity';
 import { validateEntity } from '@/GenericHelpers';
 import { ExternalHooks } from '@/ExternalHooks';
 import * as TagHelpers from '@/TagHelpers';
-import type { WorkflowRequest } from '@/requests';
+import type { ListQueryOptions, WorkflowRequest } from '@/requests';
 import type { IWorkflowDb, IWorkflowExecutionDataProcess } from '@/Interfaces';
 import { NodeTypes } from '@/NodeTypes';
 import { WorkflowRunner } from '@/WorkflowRunner';
@@ -26,25 +25,9 @@ import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData'
 import { TestWebhooks } from '@/TestWebhooks';
 import { getSharedWorkflowIds } from '@/WorkflowHelpers';
 import { isSharingEnabled, whereClause } from '@/UserManagement/UserManagementHelper';
-import type { WorkflowForList } from '@/workflows/workflows.types';
 import { InternalHooks } from '@/InternalHooks';
-
-export type IGetWorkflowsQueryFilter = Pick<
-	FindOptionsWhere<WorkflowEntity>,
-	'id' | 'name' | 'active'
->;
-
-const schemaGetWorkflowsQueryFilter = {
-	$id: '/IGetWorkflowsQueryFilter',
-	type: 'object',
-	properties: {
-		id: { anyOf: [{ type: 'integer' }, { type: 'string' }] },
-		name: { type: 'string' },
-		active: { type: 'boolean' },
-	},
-};
-
-const allowedWorkflowsQueryFilterFields = Object.keys(schemaGetWorkflowsQueryFilter.properties);
+import type { WorkflowForList } from './workflows.types';
+import { WorkflowRepository } from '@/databases/repositories';
 
 export class WorkflowsService {
 	static async getSharing(
@@ -116,71 +99,70 @@ export class WorkflowsService {
 		return getSharedWorkflowIds(user, roles);
 	}
 
-	static async getMany(user: User, rawFilter: string): Promise<WorkflowForList[]> {
+	static async getMany(
+		user: User,
+		options?: ListQueryOptions,
+	): Promise<[WorkflowForList[], number]> {
 		const sharedWorkflowIds = await this.getWorkflowIdsForUser(user, ['owner']);
 		if (sharedWorkflowIds.length === 0) {
 			// return early since without shared workflows there can be no hits
 			// (note: getSharedWorkflowIds() returns _all_ workflow ids for global owners)
-			return [];
+			return [[], 0];
 		}
 
-		let filter: IGetWorkflowsQueryFilter = {};
-		if (rawFilter) {
-			try {
-				const filterJson: JsonObject = jsonParse(rawFilter);
-				if (filterJson) {
-					Object.keys(filterJson).map((key) => {
-						if (!allowedWorkflowsQueryFilterFields.includes(key)) delete filterJson[key];
-					});
-					if (jsonSchemaValidate(filterJson, schemaGetWorkflowsQueryFilter).valid) {
-						filter = filterJson as IGetWorkflowsQueryFilter;
-					}
-				}
-			} catch (error) {
-				LoggerProxy.error('Failed to parse filter', {
-					userId: user.id,
-					filter,
-				});
-				throw new ResponseHelper.InternalServerError(
-					'Parameter "filter" contained invalid JSON string.',
-				);
-			}
-		}
+		const filter = options?.filter ?? {};
 
 		// safeguard against querying ids not shared with the user
 		const workflowId = filter?.id?.toString();
 		if (workflowId !== undefined && !sharedWorkflowIds.includes(workflowId)) {
 			LoggerProxy.verbose(`User ${user.id} attempted to query non-shared workflow ${workflowId}`);
-			return [];
+			return [[], 0];
 		}
 
-		const select: FindOptionsSelect<WorkflowEntity> = {
+		const DEFAULT_SELECT: FindOptionsSelect<WorkflowEntity> = {
 			id: true,
 			name: true,
 			active: true,
 			createdAt: true,
 			updatedAt: true,
 		};
+
+		const select: FindOptionsSelect<WorkflowEntity> = options?.select ?? DEFAULT_SELECT;
+
 		const relations: string[] = [];
 
-		if (!config.getEnv('workflowTagsDisabled')) {
+		const isDefaultSelect = options?.select === undefined;
+
+		if (isDefaultSelect && !config.getEnv('workflowTagsDisabled')) {
 			relations.push('tags');
 			select.tags = { id: true, name: true };
 		}
 
-		if (isSharingEnabled()) {
+		if (isDefaultSelect && isSharingEnabled()) {
 			relations.push('shared');
 			select.shared = { userId: true, roleId: true };
 			select.versionId = true;
 		}
 
 		filter.id = In(sharedWorkflowIds);
-		return Db.collections.Workflow.find({
+
+		if (typeof filter.name === 'string' && filter.name !== '') {
+			filter.name = Like(`%${filter.name}%`);
+		}
+
+		const findManyOptions: FindManyOptions<WorkflowEntity> = {
 			select,
 			relations,
 			where: filter,
-			order: { updatedAt: 'DESC' },
-		});
+			order: { updatedAt: 'ASC' },
+		};
+
+		if (options?.take) {
+			findManyOptions.skip = options.skip;
+			findManyOptions.take = options.take;
+		}
+
+		return Container.get(WorkflowRepository).findAndCount(findManyOptions);
 	}
 
 	static async update(