✨ Unify execution id + Queue system (#1340)

* Unify execution ID across executions

* Fix indentation and improved comments

* WIP: saving data after each node execution

* Added on/off to save data after each step, saving initial data and retries working

* Fixing lint issues

* Fixing more lint issues

* ✨ Add bull to execute workflows

* 👕 Fix lint issue

* ⚡ Add graceful shutdown to worker

* ⚡ Add loading staticData to worker

* 👕 Fix lint issue

* ⚡ Fix import

* Changed tables metadata to add nullable to stoppedAt

* Reload database on migration run

* Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration

* Added checks to Redis and exiting process if connection is unavailable

* Fixing error with new installations

* Fix issue with data not being sent back to browser on manual executions with defined destination

* Merging bull and unify execution id branch fixes

* Main process will now get execution success from database instead of redis

* Omit execution duration if execution did not stop

* Fix issue with execution list displaying inconsistant information information while a workflow is running

* Remove unused hooks to clarify for developers that these wont run in queue mode

* Added active pooling to help recover from Redis crashes

* Lint issues

* Changing default polling interval to 60 seconds

* Removed unnecessary attributes from bull job

* ⚡ Improved output on worker job start

Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>

packages/cli/src/databases/mysqldb/ExecutionEntity.ts

@@ -40,7 +40,7 @@ export class ExecutionEntity implements IExecutionFlattedDb {
 	startedAt: Date;
 
 	@Index()
-	@Column('datetime')
+	@Column('datetime', { nullable: true })
 	stoppedAt: Date;
 
 	@Column('json')

packages/cli/src/databases/mysqldb/migrations/1607431743767-MakeStoppedAtNullable.ts

@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+import * as config from '../../../../config';
+
+export class MakeStoppedAtNullable1607431743767 implements MigrationInterface {
+
+	async up(queryRunner: QueryRunner): Promise<void> {
+		const tablePrefix = config.get('database.tablePrefix');
+		await queryRunner.query('ALTER TABLE `' + tablePrefix + 'execution_entity` MODIFY `stoppedAt` datetime', undefined);
+	}
+
+	async down(queryRunner: QueryRunner): Promise<void> {
+		const tablePrefix = config.get('database.tablePrefix');
+		await queryRunner.query('ALTER TABLE `' + tablePrefix + 'execution_entity` MODIFY `stoppedAt` datetime NOT NULL', undefined);
+	}
+
+}

packages/cli/src/databases/mysqldb/migrations/index.ts

@@ -2,10 +2,12 @@ import { InitialMigration1588157391238 } from './1588157391238-InitialMigration'
 import { WebhookModel1592447867632 } from './1592447867632-WebhookModel';
 import { CreateIndexStoppedAt1594902918301 } from './1594902918301-CreateIndexStoppedAt';
 import { AddWebhookId1611149998770 } from './1611149998770-AddWebhookId';
+import { MakeStoppedAtNullable1607431743767 } from './1607431743767-MakeStoppedAtNullable';
 
 export const mysqlMigrations = [
 	InitialMigration1588157391238,
 	WebhookModel1592447867632,
 	CreateIndexStoppedAt1594902918301,
 	AddWebhookId1611149998770,
+	MakeStoppedAtNullable1607431743767,
 ];

packages/cli/src/databases/postgresdb/ExecutionEntity.ts

@@ -40,7 +40,7 @@ export class ExecutionEntity implements IExecutionFlattedDb {
 	startedAt: Date;
 
 	@Index()
-	@Column('timestamp')
+	@Column('timestamp', { nullable: true })
 	stoppedAt: Date;
 
 	@Column('json')

packages/cli/src/databases/postgresdb/migrations/1607431743768-MakeStoppedAtNullable.ts

@@ -0,0 +1,21 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+
+import * as config from '../../../../config';
+
+export class MakeStoppedAtNullable1607431743768 implements MigrationInterface {
+	name = 'MakeStoppedAtNullable1607431743768';
+
+	async up(queryRunner: QueryRunner): Promise<void> {
+		let tablePrefix = config.get('database.tablePrefix');
+		const schema = config.get('database.postgresdb.schema');
+		if (schema) {
+			tablePrefix = schema + '.' + tablePrefix;
+		}
+		await queryRunner.query('ALTER TABLE ' + tablePrefix + 'execution_entity ALTER COLUMN "stoppedAt" DROP NOT NULL', undefined);
+	}
+
+	async down(queryRunner: QueryRunner): Promise<void> {
+		// Cannot be undone as column might already have null values
+	}
+
+}

packages/cli/src/databases/postgresdb/migrations/index.ts

@@ -2,10 +2,12 @@ import { InitialMigration1587669153312 } from './1587669153312-InitialMigration'
 import { WebhookModel1589476000887 } from './1589476000887-WebhookModel';
 import { CreateIndexStoppedAt1594828256133 } from './1594828256133-CreateIndexStoppedAt';
 import { AddWebhookId1611144599516 } from './1611144599516-AddWebhookId';
+import { MakeStoppedAtNullable1607431743768 } from './1607431743768-MakeStoppedAtNullable';
 
 export const postgresMigrations = [
 	InitialMigration1587669153312,
 	WebhookModel1589476000887,
 	CreateIndexStoppedAt1594828256133,
 	AddWebhookId1611144599516,
+	MakeStoppedAtNullable1607431743768,
 ];

packages/cli/src/databases/sqlite/ExecutionEntity.ts

@@ -40,7 +40,7 @@ export class ExecutionEntity implements IExecutionFlattedDb {
 	startedAt: Date;
 
 	@Index()
-	@Column()
+	@Column({ nullable: true })
 	stoppedAt: Date;
 
 	@Column('simple-json')

packages/cli/src/databases/sqlite/migrations/1607431743769-MakeStoppedAtNullable.ts

@@ -0,0 +1,23 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+
+import * as config from '../../../../config';
+
+export class MakeStoppedAtNullable1607431743769 implements MigrationInterface {
+
+	async up(queryRunner: QueryRunner): Promise<void> {
+		const tablePrefix = config.get('database.tablePrefix');
+		// SQLite does not allow us to simply "alter column"
+		// We're hacking the way sqlite identifies tables
+		// Allowing a column to become nullable
+		// This is a very strict case when this can be done safely
+		// As no collateral effects exist.
+		await queryRunner.query(`PRAGMA writable_schema = 1; `, undefined);
+		await queryRunner.query(`UPDATE SQLITE_MASTER SET SQL = 'CREATE TABLE IF NOT EXISTS "${tablePrefix}execution_entity" ("id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "data" text NOT NULL, "finished" boolean NOT NULL, "mode" varchar NOT NULL, "retryOf" varchar, "retrySuccessId" varchar, "startedAt" datetime NOT NULL, "stoppedAt" datetime, "workflowData" text NOT NULL, "workflowId" varchar)' WHERE NAME = "${tablePrefix}execution_entity";`, undefined);
+		await queryRunner.query(`PRAGMA writable_schema = 0;`, undefined);
+	}
+
+	async down(queryRunner: QueryRunner): Promise<void> {
+		// This cannot be undone as the table might already have nullable values
+	}
+
+}

packages/cli/src/databases/sqlite/migrations/index.ts

@@ -2,10 +2,12 @@ import { InitialMigration1588102412422 } from './1588102412422-InitialMigration'
 import { WebhookModel1592445003908 } from './1592445003908-WebhookModel';
 import { CreateIndexStoppedAt1594825041918 } from './1594825041918-CreateIndexStoppedAt';
 import { AddWebhookId1611071044839 } from './1611071044839-AddWebhookId';
+import { MakeStoppedAtNullable1607431743769 } from './1607431743769-MakeStoppedAtNullable';
 
 export const sqliteMigrations = [
 	InitialMigration1588102412422,
 	WebhookModel1592445003908,
 	CreateIndexStoppedAt1594825041918,
 	AddWebhookId1611071044839,
+	MakeStoppedAtNullable1607431743769,
 ];