fix(core): Change VariablesService to DI and use caching (#6827)

* support redis cluster

* cleanup, fix config schema

* set default prefix to bull

* initial commit

* improve logging

* improve types and refactor

* list support and refactor

* fix redis service and tests

* add comment

* add redis and cache prefix

* use injection

* lint fix

* clean schema comments

* improve naming, tests, cluster client

* merge master

* cache returns unknown instead of T

* update cache service, tests and doc

* remove console.log

* VariablesService as DI, add caching, fix tests

* do not cache null or undefined values

* import fix

* more DI and remove collections

* fix merge

* lint fix

* rename to ~Cached

* fix test for CI

* fix ActiveWorkflowRunner test

packages/cli/src/environments/variables/variables.controller.ee.ts

@@ -9,6 +9,7 @@ import {
 	VariablesValidationError,
 } from './variables.service.ee';
 import { isVariablesEnabled } from './enviromentHelpers';
+import Container from 'typedi';
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
 export const EEVariablesController = express.Router();
@@ -37,7 +38,7 @@ EEVariablesController.post(
 		const variable = req.body;
 		delete variable.id;
 		try {
-			return await EEVariablesService.create(variable);
+			return await Container.get(EEVariablesService).create(variable);
 		} catch (error) {
 			if (error instanceof VariablesLicenseError) {
 				throw new ResponseHelper.BadRequestError(error.message);
@@ -63,7 +64,7 @@ EEVariablesController.patch(
 		const variable = req.body;
 		delete variable.id;
 		try {
-			return await EEVariablesService.update(id, variable);
+			return await Container.get(EEVariablesService).update(id, variable);
 		} catch (error) {
 			if (error instanceof VariablesLicenseError) {
 				throw new ResponseHelper.BadRequestError(error.message);

packages/cli/src/environments/variables/variables.controller.ts

@@ -6,6 +6,7 @@ import * as ResponseHelper from '@/ResponseHelper';
 import type { VariablesRequest } from '@/requests';
 import { VariablesService } from './variables.service';
 import { EEVariablesController } from './variables.controller.ee';
+import Container from 'typedi';
 
 export const variablesController = express.Router();
 
@@ -28,7 +29,7 @@ variablesController.use(EEVariablesController);
 variablesController.get(
 	'/',
 	ResponseHelper.send(async () => {
-		return VariablesService.getAll();
+		return Container.get(VariablesService).getAllCached();
 	}),
 );
 
@@ -43,7 +44,7 @@ variablesController.get(
 	'/:id(\\w+)',
 	ResponseHelper.send(async (req: VariablesRequest.Get) => {
 		const id = req.params.id;
-		const variable = await VariablesService.get(id);
+		const variable = await Container.get(VariablesService).getCached(id);
 		if (variable === null) {
 			throw new ResponseHelper.NotFoundError(`Variable with id ${req.params.id} not found`);
 		}
@@ -69,7 +70,7 @@ variablesController.delete(
 			});
 			throw new ResponseHelper.AuthError('Unauthorized');
 		}
-		await VariablesService.delete(id);
+		await Container.get(VariablesService).delete(id);
 
 		return true;
 	}),

packages/cli/src/environments/variables/variables.service.ee.ts

@@ -1,6 +1,5 @@
-import { Container } from 'typedi';
+import { Container, Service } from 'typedi';
 import type { Variables } from '@db/entities/Variables';
-import { collections } from '@/Db';
 import { InternalHooks } from '@/InternalHooks';
 import { generateNanoId } from '@db/utils/generators';
 import { canCreateNewVariable } from './enviromentHelpers';
@@ -9,12 +8,9 @@ import { VariablesService } from './variables.service';
 export class VariablesLicenseError extends Error {}
 export class VariablesValidationError extends Error {}
 
+@Service()
 export class EEVariablesService extends VariablesService {
-	static async getCount(): Promise<number> {
-		return collections.Variables.count();
-	}
-
-	static validateVariable(variable: Omit<Variables, 'id'>): void {
+	validateVariable(variable: Omit<Variables, 'id'>): void {
 		if (variable.key.length > 50) {
 			throw new VariablesValidationError('key cannot be longer than 50 characters');
 		}
@@ -26,23 +22,25 @@ export class EEVariablesService extends VariablesService {
 		}
 	}
 
-	static async create(variable: Omit<Variables, 'id'>): Promise<Variables> {
+	async create(variable: Omit<Variables, 'id'>): Promise<Variables> {
 		if (!canCreateNewVariable(await this.getCount())) {
 			throw new VariablesLicenseError('Variables limit reached');
 		}
 		this.validateVariable(variable);
 
 		void Container.get(InternalHooks).onVariableCreated({ variable_type: variable.type });
-		return collections.Variables.save({
+		const saveResult = await this.variablesRepository.save({
 			...variable,
 			id: generateNanoId(),
 		});
+		await this.updateCache();
+		return saveResult;
 	}
 
-	static async update(id: string, variable: Omit<Variables, 'id'>): Promise<Variables> {
+	async update(id: string, variable: Omit<Variables, 'id'>): Promise<Variables> {
 		this.validateVariable(variable);
-		await collections.Variables.update(id, variable);
-
-		return (await this.get(id))!;
+		await this.variablesRepository.update(id, variable);
+		await this.updateCache();
+		return (await this.getCached(id))!;
 	}
 }

packages/cli/src/environments/variables/variables.service.ts

@@ -1,20 +1,53 @@
 import type { Variables } from '@db/entities/Variables';
-import { collections } from '@/Db';
+import { CacheService } from '@/services/cache.service';
+import Container, { Service } from 'typedi';
+import { VariablesRepository } from '@/databases/repositories';
+import type { DeepPartial } from 'typeorm';
 
+@Service()
 export class VariablesService {
-	static async getAll(): Promise<Variables[]> {
-		return collections.Variables.find();
+	constructor(
+		protected cacheService: CacheService,
+		protected variablesRepository: VariablesRepository,
+	) {}
+
+	async getAllCached(): Promise<Variables[]> {
+		const variables = await this.cacheService.get('variables', {
+			async refreshFunction() {
+				// TODO: log refresh cache metric
+				return Container.get(VariablesService).findAll();
+			},
+		});
+		return (variables as Array<DeepPartial<Variables>>).map((v) =>
+			this.variablesRepository.create(v),
+		);
 	}
 
-	static async getCount(): Promise<number> {
-		return collections.Variables.count();
+	async getCount(): Promise<number> {
+		return (await this.getAllCached()).length;
 	}
 
-	static async get(id: string): Promise<Variables | null> {
-		return collections.Variables.findOne({ where: { id } });
+	async getCached(id: string): Promise<Variables | null> {
+		const variables = await this.getAllCached();
+		const foundVariable = variables.find((variable) => variable.id === id);
+		if (!foundVariable) {
+			return null;
+		}
+		return this.variablesRepository.create(foundVariable as DeepPartial<Variables>);
 	}
 
-	static async delete(id: string): Promise<void> {
-		await collections.Variables.delete(id);
+	async delete(id: string): Promise<void> {
+		await this.variablesRepository.delete(id);
+		await this.updateCache();
+	}
+
+	async updateCache(): Promise<void> {
+		// TODO: log update cache metric
+		const variables = await this.findAll();
+		await this.cacheService.set('variables', variables);
+	}
+
+	async findAll(): Promise<Variables[]> {
+		return this.variablesRepository.find();
 	}
 }