perf(core): Add filtering and pagination to GET /workflows (#6845)

* Initial setup

* Specify max paginated items

* Simplify

* Add tests

* Add more tests

* Add migrations

* Add top-level property

* Add field selection

* Cleanup

* Rename `total` to `count`

* More cleanup

* Move query logic into `WorkflowRepository`

* Create `AbstractRepository`

* Cleanup

* Fix name

* Remove leftover comments

* Replace reference

* Add default for `rawSkip`

* Remove unneeded typing

* Switch to `class-validator`

* Simplify

* Simplify

* Type as optional

* Make typing more accurate

* Fix lint

* Use `getOwnPropertyNames`

* Use DSL

* Set schema at repo level

* Cleanup

* Remove comment

* Refactor repository methods to middleware

* Add middleware tests

* Remove old test files

* Remove generic experiment

* Reuse `reportError`

* Remove unused type

* Cleanup

* Improve wording

* Reduce diff

* Add missing mw

* Use `Container.get`

* Adjust lint rule

* Reorganize into subdir

* Remove unused directive

* Remove nodes

* Silly mistake

* Validate take

* refactor(core): Adjust index handling in new migrations DSL (no-changelog) (#6876)

* refactor(core): Adjust index handling in new migrations DSL (no-changelog)

* Account for custom index name

* Also for dropping

* Fix `select` issue with `relations`

* Tighten validation

* Ensure `ownerId` is not added when specifying `select`

packages/cli/src/middlewares/index.ts

@@ -1,3 +1,4 @@
 export * from './auth';
 export * from './bodyParser';
 export * from './cors';
+export * from './listQuery';

packages/cli/src/middlewares/listQuery/error.ts

@@ -0,0 +1,21 @@
+import { BadRequestError } from '@/ResponseHelper';
+import { LoggerProxy } from 'n8n-workflow';
+import * as utils from '@/utils';
+
+export function handleListQueryError(
+	paramName: 'filter' | 'select',
+	paramValue: string,
+	maybeError: unknown,
+) {
+	const error = utils.toError(maybeError);
+
+	LoggerProxy.error(`Invalid "${paramName}" query parameter`, {
+		paramName,
+		paramValue,
+		error,
+	});
+
+	throw new BadRequestError(
+		`Invalid "${paramName}" query parameter: ${paramValue}. Error: ${error.message}`,
+	);
+}

packages/cli/src/middlewares/listQuery/filter.ts

@@ -0,0 +1,44 @@
+import { jsonParse } from 'n8n-workflow';
+import { handleListQueryError } from './error';
+import { WorkflowSchema } from './workflow.schema';
+import type { ListQueryRequest } from '@/requests';
+import type { RequestHandler } from 'express';
+import type { Schema } from './schema';
+
+function toQueryFilter(rawFilter: string, schema: typeof Schema) {
+	const parsedFilter = new schema(
+		jsonParse(rawFilter, { errorMessage: 'Failed to parse filter JSON' }),
+	);
+
+	return Object.fromEntries(
+		Object.entries(parsedFilter)
+			.filter(([_, value]) => value !== undefined)
+			.map(([key, _]: [keyof Schema, unknown]) => [key, parsedFilter[key]]),
+	);
+}
+
+export const filterListQueryMiddleware: RequestHandler = (req: ListQueryRequest, res, next) => {
+	const { filter: rawFilter } = req.query;
+
+	if (!rawFilter) return next();
+
+	let schema;
+
+	if (req.baseUrl.endsWith('workflows')) {
+		schema = WorkflowSchema;
+	} else {
+		return next();
+	}
+
+	try {
+		const filter = toQueryFilter(rawFilter, schema);
+
+		if (Object.keys(filter).length === 0) return next();
+
+		req.listQueryOptions = { ...req.listQueryOptions, filter };
+
+		next();
+	} catch (error) {
+		handleListQueryError('filter', rawFilter, error);
+	}
+};

packages/cli/src/middlewares/listQuery/index.ts

@@ -0,0 +1,9 @@
+import { filterListQueryMiddleware } from './filter';
+import { selectListQueryMiddleware } from './select';
+import { paginationListQueryMiddleware } from './pagination';
+
+export const listQueryMiddleware = [
+	filterListQueryMiddleware,
+	selectListQueryMiddleware,
+	paginationListQueryMiddleware,
+];

packages/cli/src/middlewares/listQuery/pagination.ts

@@ -0,0 +1,27 @@
+import type { ListQueryRequest } from '@/requests';
+import { isIntegerString } from '@/utils';
+import type { RequestHandler } from 'express';
+
+function toPaginationOptions(rawTake: string, rawSkip: string) {
+	const MAX_ITEMS = 50;
+
+	if ([rawTake, rawSkip].some((i) => !isIntegerString(i))) {
+		throw new Error('Parameter take or skip is not an integer string');
+	}
+
+	const [take, skip] = [rawTake, rawSkip].map((o) => parseInt(o, 10));
+
+	return { skip, take: Math.min(take, MAX_ITEMS) };
+}
+
+export const paginationListQueryMiddleware: RequestHandler = (req: ListQueryRequest, res, next) => {
+	const { take: rawTake, skip: rawSkip = '0' } = req.query;
+
+	if (!rawTake) return next();
+
+	const { take, skip } = toPaginationOptions(rawTake, rawSkip);
+
+	req.listQueryOptions = { ...req.listQueryOptions, take, skip };
+
+	next();
+};

packages/cli/src/middlewares/listQuery/schema.ts

@@ -0,0 +1,7 @@
+export class Schema {
+	constructor(private data: unknown = {}) {}
+
+	static get fieldNames(): string[] {
+		return [];
+	}
+}

packages/cli/src/middlewares/listQuery/select.ts

@@ -0,0 +1,46 @@
+import { handleListQueryError } from './error';
+import { jsonParse } from 'n8n-workflow';
+import { WorkflowSchema } from './workflow.schema';
+import * as utils from '@/utils';
+import type { ListQueryRequest } from '@/requests';
+import type { RequestHandler } from 'express';
+import type { Schema } from '@/middlewares/listQuery/schema';
+
+function toQuerySelect(rawSelect: string, schema: typeof Schema) {
+	const asArr = jsonParse(rawSelect, { errorMessage: 'Failed to parse select JSON' });
+
+	if (!utils.isStringArray(asArr)) {
+		throw new Error('Parsed select is not a string array');
+	}
+
+	return asArr.reduce<Record<string, true>>((acc, field) => {
+		if (!schema.fieldNames.includes(field)) return acc;
+		return (acc[field] = true), acc;
+	}, {});
+}
+
+export const selectListQueryMiddleware: RequestHandler = (req: ListQueryRequest, res, next) => {
+	const { select: rawSelect } = req.query;
+
+	if (!rawSelect) return next();
+
+	let schema;
+
+	if (req.baseUrl.endsWith('workflows')) {
+		schema = WorkflowSchema;
+	} else {
+		return next();
+	}
+
+	try {
+		const select = toQuerySelect(rawSelect, schema);
+
+		if (Object.keys(select).length === 0) return next();
+
+		req.listQueryOptions = { ...req.listQueryOptions, select };
+
+		next();
+	} catch (error) {
+		handleListQueryError('select', rawSelect, error);
+	}
+};

packages/cli/src/middlewares/listQuery/workflow.schema.ts

@@ -0,0 +1,40 @@
+import { Schema } from '@/middlewares/listQuery/schema';
+import { validateSync, IsOptional, IsString, IsBoolean, IsDateString } from 'class-validator';
+
+export class WorkflowSchema extends Schema {
+	constructor(data: unknown = {}) {
+		super();
+		Object.assign(this, data);
+
+		// strip out unknown fields
+		const result = validateSync(this, { whitelist: true });
+
+		if (result.length > 0) {
+			throw new Error('Parsed filter does not fit the schema');
+		}
+	}
+
+	@IsOptional()
+	@IsString()
+	id?: string = undefined;
+
+	@IsOptional()
+	@IsString()
+	name?: string = undefined;
+
+	@IsOptional()
+	@IsBoolean()
+	active?: boolean = undefined;
+
+	@IsOptional()
+	@IsDateString()
+	createdAt?: Date = undefined;
+
+	@IsOptional()
+	@IsDateString()
+	updatedAt?: Date = undefined;
+
+	static get fieldNames() {
+		return Object.getOwnPropertyNames(new WorkflowSchema());
+	}
+}