From 05507fc19b848b8fd8f293b817da734154210d74 Mon Sep 17 00:00:00 2001 From: Ben Hesseldieck Date: Thu, 9 Jul 2020 15:17:47 +0200 Subject: [PATCH 1/3] :tada: basic throtteling with cleaning logs --- .../cli/src/WorkflowExecuteAdditionalData.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index ed20a985f..86e5c3fe2 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -41,6 +41,8 @@ import { import * as config from '../config'; +import { LessThanOrEqual } from "typeorm"; + /** * Checks if there was an error and if errorWorkflow is defined. If so it collects @@ -79,6 +81,25 @@ function executeErrorWorkflow(workflowData: IWorkflowBase, fullRunData: IRun, mo } } +/** + * Prunes Saved Execution which are older than configured. + * Throttled to be executed just once in configured timeframe. + * + */ +let inThrottle: boolean; +function pruneSavedExecutions(): void { + console.log('THROTTLE:', inThrottle); + if (!inThrottle) { + inThrottle = true; + Db.collections.Execution!.delete({ startedAt: LessThanOrEqual(new Date().toISOString()) }); + console.log('Deleting logs'); + setTimeout(() => { + console.log('resetting throttle'); + inThrottle = false; + }, 30000); + } +} + /** * Pushes the execution out to all connected clients @@ -251,6 +272,7 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { // Save the Execution in DB const executionResult = await Db.collections.Execution!.save(executionData as IExecutionFlattedDb); + pruneSavedExecutions() if (fullRunData.finished === true && this.retryOf !== undefined) { // If the retry was successful save the reference it on the original execution From b956444c0e41c24c426daaac91e597093c7ef129 Mon Sep 17 00:00:00 2001 From: Ben Hesseldieck Date: Thu, 9 Jul 2020 16:52:38 +0200 Subject: [PATCH 2/3] :racehorse: pruning execution data complete --- packages/cli/config/index.ts | 25 ++++++++++++-- .../cli/src/WorkflowExecuteAdditionalData.ts | 33 ++++++++++++------- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/packages/cli/config/index.ts b/packages/cli/config/index.ts index 847587460..0022c08b7 100644 --- a/packages/cli/config/index.ts +++ b/packages/cli/config/index.ts @@ -161,8 +161,8 @@ const config = convict({ // If a workflow executes all the data gets saved by default. This // could be a problem when a workflow gets executed a lot and processes - // a lot of data. To not write the database full it is possible to - // not save the execution at all. + // a lot of data. To not exceed the database's capacity it is possible to + // prune the database regularly or to not save the execution at all. // Depending on if the execution did succeed or error a different // save behaviour can be set. saveDataOnError: { @@ -188,6 +188,27 @@ const config = convict({ default: false, env: 'EXECUTIONS_DATA_SAVE_MANUAL_EXECUTIONS' }, + + // To not exceed the database's capacity and keep its size moderate + // the execution data gets pruned regularly (default: 1 hour interval). + // All saved execution data older than the max age will be deleted. + // Pruning is currently not activated by default, which will change in + // a future version. + pruneData: { + doc: 'Delete data of past executions on a rolling basis', + default: false, + env: 'EXECUTIONS_DATA_PRUNE' + }, + pruneDataMaxAge: { + doc: 'How old (hours) the execution data has to be to get deleted', + default: 336, + env: 'EXECUTIONS_DATA_MAX_AGE' + }, + pruneDataTimeout: { + doc: 'Timeout (ms) after execution data has been pruned', + default: 3600000, + env: 'EXECUTIONS_DATA_PRUNE_TIMEOUT' + }, }, generic: { diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index 86e5c3fe2..9b55f88b5 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -86,17 +86,22 @@ function executeErrorWorkflow(workflowData: IWorkflowBase, fullRunData: IRun, mo * Throttled to be executed just once in configured timeframe. * */ -let inThrottle: boolean; -function pruneSavedExecutions(): void { - console.log('THROTTLE:', inThrottle); - if (!inThrottle) { - inThrottle = true; - Db.collections.Execution!.delete({ startedAt: LessThanOrEqual(new Date().toISOString()) }); - console.log('Deleting logs'); - setTimeout(() => { - console.log('resetting throttle'); - inThrottle = false; - }, 30000); +let throttling: boolean; +function pruneExecutionData(): void { + if (!throttling) { + throttling = true; + const timeout = config.get('executions.pruneDataTimeout') as number; // in ms + const maxAge = config.get('executions.pruneDataMaxAge') as number; // in h + const date = new Date(); // today + date.setHours(date.getHours() - maxAge); + + // throttle just on success to allow for self healing on failure + Db.collections.Execution!.delete({ startedAt: LessThanOrEqual(date.toISOString()) }) + .then(data => + setTimeout(() => { + throttling = false; + }, timeout) + ).catch(err => throttling = false) } } @@ -272,7 +277,6 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { // Save the Execution in DB const executionResult = await Db.collections.Execution!.save(executionData as IExecutionFlattedDb); - pruneSavedExecutions() if (fullRunData.finished === true && this.retryOf !== undefined) { // If the retry was successful save the reference it on the original execution @@ -280,6 +284,11 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { await Db.collections.Execution!.update(this.retryOf, { retrySuccessId: executionResult.id }); } + // Prune old execution data + if (config.get('executions.pruneData')) { + pruneExecutionData() + } + if (!isManualMode) { executeErrorWorkflow(this.workflowData, fullRunData, this.mode, executionResult ? executionResult.id as string : undefined, this.retryOf); } From 8bfc5a4b65803cd9afef1fce106f13968eb5685d Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Sun, 12 Jul 2020 12:48:32 +0200 Subject: [PATCH 3/3] :zap: Small improvements on execution pruning --- packages/cli/config/index.ts | 4 ++-- .../cli/src/WorkflowExecuteAdditionalData.ts | 20 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/cli/config/index.ts b/packages/cli/config/index.ts index 239061746..9492b4cb6 100644 --- a/packages/cli/config/index.ts +++ b/packages/cli/config/index.ts @@ -205,8 +205,8 @@ const config = convict({ env: 'EXECUTIONS_DATA_MAX_AGE' }, pruneDataTimeout: { - doc: 'Timeout (ms) after execution data has been pruned', - default: 3600000, + doc: 'Timeout (seconds) after execution data has been pruned', + default: 3600, env: 'EXECUTIONS_DATA_PRUNE_TIMEOUT' }, }, diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index 9b55f88b5..4230e79f5 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -86,22 +86,22 @@ function executeErrorWorkflow(workflowData: IWorkflowBase, fullRunData: IRun, mo * Throttled to be executed just once in configured timeframe. * */ -let throttling: boolean; +let throttling = false; function pruneExecutionData(): void { if (!throttling) { throttling = true; - const timeout = config.get('executions.pruneDataTimeout') as number; // in ms + const timeout = config.get('executions.pruneDataTimeout') as number; // in seconds const maxAge = config.get('executions.pruneDataMaxAge') as number; // in h const date = new Date(); // today date.setHours(date.getHours() - maxAge); // throttle just on success to allow for self healing on failure - Db.collections.Execution!.delete({ startedAt: LessThanOrEqual(date.toISOString()) }) + Db.collections.Execution!.delete({ stoppedAt: LessThanOrEqual(date.toISOString()) }) .then(data => setTimeout(() => { throttling = false; - }, timeout) - ).catch(err => throttling = false) + }, timeout * 1000) + ).catch(err => throttling = false); } } @@ -215,6 +215,11 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { workflowExecuteAfter: [ async function (this: WorkflowHooks, fullRunData: IRun, newStaticData: IDataObject): Promise { + // Prune old execution data + if (config.get('executions.pruneData')) { + pruneExecutionData(); + } + const isManualMode = [this.mode, parentProcessMode].includes('manual'); try { @@ -284,11 +289,6 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { await Db.collections.Execution!.update(this.retryOf, { retrySuccessId: executionResult.id }); } - // Prune old execution data - if (config.get('executions.pruneData')) { - pruneExecutionData() - } - if (!isManualMode) { executeErrorWorkflow(this.workflowData, fullRunData, this.mode, executionResult ? executionResult.id as string : undefined, this.retryOf); }