✨ Unify execution id + Queue system (#1340)

* Unify execution ID across executions

* Fix indentation and improved comments

* WIP: saving data after each node execution

* Added on/off to save data after each step, saving initial data and retries working

* Fixing lint issues

* Fixing more lint issues

* ✨ Add bull to execute workflows

* 👕 Fix lint issue

* ⚡ Add graceful shutdown to worker

* ⚡ Add loading staticData to worker

* 👕 Fix lint issue

* ⚡ Fix import

* Changed tables metadata to add nullable to stoppedAt

* Reload database on migration run

* Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration

* Added checks to Redis and exiting process if connection is unavailable

* Fixing error with new installations

* Fix issue with data not being sent back to browser on manual executions with defined destination

* Merging bull and unify execution id branch fixes

* Main process will now get execution success from database instead of redis

* Omit execution duration if execution did not stop

* Fix issue with execution list displaying inconsistant information information while a workflow is running

* Remove unused hooks to clarify for developers that these wont run in queue mode

* Added active pooling to help recover from Redis crashes

* Lint issues

* Changing default polling interval to 60 seconds

* Removed unnecessary attributes from bull job

* ⚡ Improved output on worker job start

Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>

packages/cli/config/index.ts

@@ -159,6 +159,13 @@ const config = convict({
 			env: 'EXECUTIONS_PROCESS',
 		},
 
+		mode: {
+			doc: 'If it should run executions directly or via queue',
+			format: ['regular', 'queue'],
+			default: 'regular',
+			env: 'EXECUTIONS_MODE',
+		},
+
 		// A Workflow times out and gets canceled after this time (seconds).
 		// If the workflow is executed in the main process a soft timeout
 		// is executed (takes effect after the current node finishes).
@@ -201,6 +208,12 @@ const config = convict({
 			default: 'all',
 			env: 'EXECUTIONS_DATA_SAVE_ON_SUCCESS',
 		},
+		saveExecutionProgress: {
+			doc: 'Wether or not to save progress for each node executed',
+			format: 'Boolean',
+			default: false,
+			env: 'EXECUTIONS_DATA_SAVE_ON_PROGRESS',
+		},
 
 		// If the executions of workflows which got started via the editor
 		// should be saved. By default they will not be saved as this runs
@@ -239,6 +252,54 @@ const config = convict({
 		},
 	},
 
+	queue: {
+		bull: {
+			prefix: {
+				doc: 'Prefix for all queue keys',
+				format: String,
+				default: '',
+				env: 'QUEUE_BULL_PREFIX',
+			},
+			redis: {
+				db: {
+					doc: 'Redis DB',
+					format: Number,
+					default: 0,
+					env: 'QUEUE_BULL_REDIS_DB',
+				},
+				host: {
+					doc: 'Redis Host',
+					format: String,
+					default: 'localhost',
+					env: 'QUEUE_BULL_REDIS_HOST',
+				},
+				password: {
+					doc: 'Redis Password',
+					format: String,
+					default: '',
+					env: 'QUEUE_BULL_REDIS_PASSWORD',
+				},
+				port: {
+					doc: 'Redis Port',
+					format: Number,
+					default: 6379,
+					env: 'QUEUE_BULL_REDIS_PORT',
+				},
+				timeoutThreshold: {
+					doc: 'Redis timeout threshold',
+					format: Number,
+					default: 10000,
+					env: 'QUEUE_BULL_REDIS_TIMEOUT_THRESHOLD',
+				},
+			},
+			queueRecoveryInterval: {
+				doc: 'If > 0 enables an active polling to the queue that can recover for Redis crashes. Given in seconds; 0 is disabled. May increase Redis traffic significantly.',
+				format: Number,
+				default: 60,
+				env: 'QUEUE_RECOVERY_INTERVAL',
+			},
+		},
+	},
 	generic: {
 		// The timezone to use. Is important for nodes like "Cron" which start the
 		// workflow automatically at a specified time. This setting can also be