perf(core): Add filtering and pagination to GET /workflows (#6845)

* Initial setup

* Specify max paginated items

* Simplify

* Add tests

* Add more tests

* Add migrations

* Add top-level property

* Add field selection

* Cleanup

* Rename `total` to `count`

* More cleanup

* Move query logic into `WorkflowRepository`

* Create `AbstractRepository`

* Cleanup

* Fix name

* Remove leftover comments

* Replace reference

* Add default for `rawSkip`

* Remove unneeded typing

* Switch to `class-validator`

* Simplify

* Simplify

* Type as optional

* Make typing more accurate

* Fix lint

* Use `getOwnPropertyNames`

* Use DSL

* Set schema at repo level

* Cleanup

* Remove comment

* Refactor repository methods to middleware

* Add middleware tests

* Remove old test files

* Remove generic experiment

* Reuse `reportError`

* Remove unused type

* Cleanup

* Improve wording

* Reduce diff

* Add missing mw

* Use `Container.get`

* Adjust lint rule

* Reorganize into subdir

* Remove unused directive

* Remove nodes

* Silly mistake

* Validate take

* refactor(core): Adjust index handling in new migrations DSL (no-changelog) (#6876)

* refactor(core): Adjust index handling in new migrations DSL (no-changelog)

* Account for custom index name

* Also for dropping

* Fix `select` issue with `relations`

* Tighten validation

* Ensure `ownerId` is not added when specifying `select`

packages/cli/src/databases/dsl/Indices.ts

@@ -4,11 +4,20 @@ import LazyPromise from 'p-lazy';
 abstract class IndexOperation extends LazyPromise<void> {
 	abstract execute(queryRunner: QueryRunner): Promise<void>;
 
+	get fullTableName() {
+		return [this.tablePrefix, this.tableName].join('');
+	}
+
+	get fullIndexName() {
+		return ['IDX', `${this.tablePrefix}${this.tableName}`, ...this.columnNames].join('_');
+	}
+
 	constructor(
-		protected name: string,
+		protected tablePrefix: string,
 		protected tableName: string,
-		protected prefix: string,
+		protected columnNames: string[],
 		queryRunner: QueryRunner,
+		protected customIndexName?: string,
 	) {
 		super((resolve) => {
 			void this.execute(queryRunner).then(resolve);
@@ -18,28 +27,27 @@ abstract class IndexOperation extends LazyPromise<void> {
 
 export class CreateIndex extends IndexOperation {
 	constructor(
-		name: string,
+		tablePrefix: string,
 		tableName: string,
-		protected columnNames: string[],
+		columnNames: string[],
 		protected isUnique: boolean,
-		prefix: string,
 		queryRunner: QueryRunner,
+		customIndexName?: string,
 	) {
-		super(name, tableName, prefix, queryRunner);
+		super(tablePrefix, tableName, columnNames, queryRunner, customIndexName);
 	}
 
 	async execute(queryRunner: QueryRunner) {
-		const { tableName, name, columnNames, prefix, isUnique } = this;
+		const { columnNames, isUnique } = this;
 		return queryRunner.createIndex(
-			`${prefix}${tableName}`,
-			new TableIndex({ name: `IDX_${prefix}${name}`, columnNames, isUnique }),
+			this.fullTableName,
+			new TableIndex({ name: this.customIndexName ?? this.fullIndexName, columnNames, isUnique }),
 		);
 	}
 }
 
 export class DropIndex extends IndexOperation {
 	async execute(queryRunner: QueryRunner) {
-		const { tableName, name, prefix } = this;
-		return queryRunner.dropIndex(`${prefix}${tableName}`, `IDX_${prefix}${name}`);
+		return queryRunner.dropIndex(this.fullTableName, this.customIndexName ?? this.fullIndexName);
 	}
 }

packages/cli/src/databases/dsl/index.ts

@@ -7,11 +7,19 @@ export const createSchemaBuilder = (tablePrefix: string, queryRunner: QueryRunne
 	column: (name: string) => new Column(name),
 	/* eslint-disable @typescript-eslint/promise-function-async */
 	// NOTE: Do not add `async` to these functions, as that messes up the lazy-evaluation of LazyPromise
-	createTable: (name: string) => new CreateTable(name, tablePrefix, queryRunner),
-	dropTable: (name: string) => new DropTable(name, tablePrefix, queryRunner),
-	createIndex: (name: string, tableName: string, columnNames: string[], isUnique = false) =>
-		new CreateIndex(name, tableName, columnNames, isUnique, tablePrefix, queryRunner),
-	dropIndex: (name: string, tableName: string) =>
-		new DropIndex(name, tableName, tablePrefix, queryRunner),
+	createTable: (tableName: string) => new CreateTable(tableName, tablePrefix, queryRunner),
+
+	dropTable: (tableName: string) => new DropTable(tableName, tablePrefix, queryRunner),
+
+	createIndex: (
+		tableName: string,
+		columnNames: string[],
+		isUnique = false,
+		customIndexName?: string,
+	) => new CreateIndex(tablePrefix, tableName, columnNames, isUnique, queryRunner, customIndexName),
+
+	dropIndex: (tableName: string, columnNames: string[], customIndexName?: string) =>
+		new DropIndex(tablePrefix, tableName, columnNames, queryRunner, customIndexName),
+
 	/* eslint-enable */
 });

packages/cli/src/databases/migrations/common/1691088862123-CreateWorkflowNameIndex.ts

@@ -0,0 +1,11 @@
+import type { MigrationContext, ReversibleMigration } from '@db/types';
+
+export class CreateWorkflowNameIndex1691088862123 implements ReversibleMigration {
+	async up({ schemaBuilder: { createIndex } }: MigrationContext) {
+		await createIndex('workflow_entity', ['name']);
+	}
+
+	async down({ schemaBuilder: { dropIndex } }: MigrationContext) {
+		await dropIndex('workflow_entity', ['name']);
+	}
+}

packages/cli/src/databases/migrations/mysqldb/index.ts

@@ -43,6 +43,7 @@ import { SeparateExecutionData1690000000030 } from './1690000000030-SeparateExec
 import { FixExecutionDataType1690000000031 } from './1690000000031-FixExecutionDataType';
 import { RemoveSkipOwnerSetup1681134145997 } from './1681134145997-RemoveSkipOwnerSetup';
 import { RemoveResetPasswordColumns1690000000030 } from '../common/1690000000030-RemoveResetPasswordColumns';
+import { CreateWorkflowNameIndex1691088862123 } from '../common/1691088862123-CreateWorkflowNameIndex';
 
 export const mysqlMigrations: Migration[] = [
 	InitialMigration1588157391238,
@@ -89,4 +90,5 @@ export const mysqlMigrations: Migration[] = [
 	FixExecutionDataType1690000000031,
 	RemoveSkipOwnerSetup1681134145997,
 	RemoveResetPasswordColumns1690000000030,
+	CreateWorkflowNameIndex1691088862123,
 ];

packages/cli/src/databases/migrations/postgresdb/index.ts

@@ -41,6 +41,7 @@ import { SeparateExecutionData1690000000020 } from './1690000000020-SeparateExec
 import { RemoveSkipOwnerSetup1681134145997 } from './1681134145997-RemoveSkipOwnerSetup';
 import { RemoveResetPasswordColumns1690000000030 } from '../common/1690000000030-RemoveResetPasswordColumns';
 import { AddMissingPrimaryKeyOnExecutionData1690787606731 } from './1690787606731-AddMissingPrimaryKeyOnExecutionData';
+import { CreateWorkflowNameIndex1691088862123 } from '../common/1691088862123-CreateWorkflowNameIndex';
 
 export const postgresMigrations: Migration[] = [
 	InitialMigration1587669153312,
@@ -85,4 +86,5 @@ export const postgresMigrations: Migration[] = [
 	RemoveSkipOwnerSetup1681134145997,
 	RemoveResetPasswordColumns1690000000030,
 	AddMissingPrimaryKeyOnExecutionData1690787606731,
+	CreateWorkflowNameIndex1691088862123,
 ];

packages/cli/src/databases/migrations/sqlite/index.ts

@@ -40,6 +40,7 @@ import { SeparateExecutionData1690000000010 } from './1690000000010-SeparateExec
 import { RemoveSkipOwnerSetup1681134145997 } from './1681134145997-RemoveSkipOwnerSetup';
 import { FixMissingIndicesFromStringIdMigration1690000000020 } from './1690000000020-FixMissingIndicesFromStringIdMigration';
 import { RemoveResetPasswordColumns1690000000030 } from './1690000000030-RemoveResetPasswordColumns';
+import { CreateWorkflowNameIndex1691088862123 } from '../common/1691088862123-CreateWorkflowNameIndex';
 
 const sqliteMigrations: Migration[] = [
 	InitialMigration1588102412422,
@@ -83,6 +84,7 @@ const sqliteMigrations: Migration[] = [
 	RemoveSkipOwnerSetup1681134145997,
 	FixMissingIndicesFromStringIdMigration1690000000020,
 	RemoveResetPasswordColumns1690000000030,
+	CreateWorkflowNameIndex1691088862123,
 ];
 
 export { sqliteMigrations };