From 2137ae23d789af765a7555028e9651029f88cadc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E0=A4=95=E0=A4=BE=E0=A4=B0=E0=A4=A4=E0=A5=8B=E0=A4=AB?=
 =?UTF-8?q?=E0=A5=8D=E0=A4=AB=E0=A5=87=E0=A4=B2=E0=A4=B8=E0=A5=8D=E0=A4=95?=
 =?UTF-8?q?=E0=A5=8D=E0=A4=B0=E0=A4=BF=E0=A4=AA=E0=A5=8D=E0=A4=9F=E2=84=A2?=
 <netroy@users.noreply.github.com>
Date: Fri, 24 Feb 2023 18:02:34 +0100
Subject: [PATCH] fix(core): Fix execution pruning queries (#5562)

* fix(core): Execution pruning should delete query should use the `OR` operator

* fix(core): Prune executions in a chunk to avoid sqlite error "Expression tree is too large"

* reduce the memory usage during execution pruning
---
 .../cli/src/WorkflowExecuteAdditionalData.ts  | 34 +++++++++++--------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts
index cf21a80ca..4488af126 100644
--- a/packages/cli/src/WorkflowExecuteAdditionalData.ts
+++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts
@@ -44,7 +44,7 @@ import {
 
 import pick from 'lodash.pick';
 import type { FindOptionsWhere } from 'typeorm';
-import { LessThanOrEqual } from 'typeorm';
+import { LessThanOrEqual, In } from 'typeorm';
 import { DateUtils } from 'typeorm/util/DateUtils';
 import config from '@/config';
 import * as Db from '@/Db';
@@ -212,7 +212,9 @@ async function pruneExecutionData(this: WorkflowHooks): Promise<void> {
 		// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
 		const utcDate = DateUtils.mixedDateToUtcDatetimeString(date);
 
-		const toPrune: FindOptionsWhere<IExecutionFlattedDb> = { stoppedAt: LessThanOrEqual(utcDate) };
+		const toPrune: Array<FindOptionsWhere<IExecutionFlattedDb>> = [
+			{ stoppedAt: LessThanOrEqual(utcDate) },
+		];
 
 		if (maxCount > 0) {
 			const executions = await Db.collections.Execution.find({
@@ -223,27 +225,29 @@ async function pruneExecutionData(this: WorkflowHooks): Promise<void> {
 			});
 
 			if (executions[0]) {
-				toPrune.id = LessThanOrEqual(executions[0].id);
+				toPrune.push({ id: LessThanOrEqual(executions[0].id) });
 			}
 		}
 
 		const isBinaryModeDefaultMode = config.getEnv('binaryDataManager.mode') === 'default';
 		try {
-			const executions = isBinaryModeDefaultMode
-				? []
-				: await Db.collections.Execution.find({
-						select: ['id'],
-						where: toPrune,
-				  });
-			await Db.collections.Execution.delete(toPrune);
 			setTimeout(() => {
 				throttling = false;
 			}, timeout * 1000);
-			// Mark binary data for deletion for all executions
-			if (!isBinaryModeDefaultMode)
-				await BinaryDataManager.getInstance().markDataForDeletionByExecutionIds(
-					executions.map(({ id }) => id),
-				);
+			let executionIds: Array<IExecutionFlattedDb['id']>;
+			do {
+				executionIds = (
+					await Db.collections.Execution.find({
+						select: ['id'],
+						where: toPrune,
+						take: 100,
+					})
+				).map(({ id }) => id);
+				await Db.collections.Execution.delete({ id: In(executionIds) });
+				// Mark binary data for deletion for all executions
+				if (!isBinaryModeDefaultMode)
+					await BinaryDataManager.getInstance().markDataForDeletionByExecutionIds(executionIds);
+			} while (executionIds.length > 0);
 		} catch (error) {
 			ErrorReporter.error(error);
 			throttling = false;