feat(core): Make Redis available for backend communication (#6719)
* support redis cluster * cleanup, fix config schema * set default prefix to bull * initial commit * improve logging * improve types and refactor * list support and refactor * fix redis service and tests * add comment * add redis and cache prefix * use injection * lint fix * clean schema comments * improve naming, tests, cluster client * merge master * cache returns unknown instead of T * update cache service, tests and doc * remove console.log * do not cache null or undefined values * fix merge * lint fix
This commit is contained in:
Michael Auerswald
committed by
GitHub
GitHub
parent
4ac4b850dd
commit
3cad60e918
@@ -1,18 +1,59 @@
|
||||
import Container from 'typedi';
|
||||
import { CacheService } from '@/services/cache.service';
|
||||
import type { MemoryCache } from 'cache-manager';
|
||||
// import type { RedisCache } from 'cache-manager-ioredis-yet';
|
||||
import type { RedisCache } from 'cache-manager-ioredis-yet';
|
||||
import config from '@/config';
|
||||
import { LoggerProxy } from 'n8n-workflow';
|
||||
import { getLogger } from '@/Logger';
|
||||
|
||||
const cacheService = Container.get(CacheService);
|
||||
|
||||
function setDefaultConfig() {
|
||||
config.set('executions.mode', 'regular');
|
||||
config.set('cache.backend', 'auto');
|
||||
config.set('cache.enabled', true);
|
||||
config.set('cache.backend', 'memory');
|
||||
config.set('cache.memory.maxSize', 1 * 1024 * 1024);
|
||||
}
|
||||
|
||||
interface TestObject {
|
||||
test: string;
|
||||
test2: number;
|
||||
test3?: TestObject & { test4: TestObject };
|
||||
}
|
||||
|
||||
const testObject: TestObject = {
|
||||
test: 'test',
|
||||
test2: 123,
|
||||
test3: {
|
||||
test: 'test3',
|
||||
test2: 123,
|
||||
test4: {
|
||||
test: 'test4',
|
||||
test2: 123,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
describe('cacheService', () => {
|
||||
beforeAll(async () => {
|
||||
LoggerProxy.init(getLogger());
|
||||
jest.mock('ioredis', () => {
|
||||
const Redis = require('ioredis-mock');
|
||||
if (typeof Redis === 'object') {
|
||||
// the first mock is an ioredis shim because ioredis-mock depends on it
|
||||
// https://github.com/stipsan/ioredis-mock/blob/master/src/index.js#L101-L111
|
||||
return {
|
||||
Command: { _transformer: { argument: {}, reply: {} } },
|
||||
};
|
||||
}
|
||||
// second mock for our code
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return function (...args: any) {
|
||||
return new Redis(args);
|
||||
};
|
||||
});
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
setDefaultConfig();
|
||||
await Container.get(CacheService).destroy();
|
||||
@@ -29,43 +70,43 @@ describe('cacheService', () => {
|
||||
test('should cache and retrieve a value', async () => {
|
||||
await cacheService.init();
|
||||
await expect(cacheService.getCache()).resolves.toBeDefined();
|
||||
await cacheService.set<string>('testString', 'test');
|
||||
await cacheService.set<number>('testNumber', 123);
|
||||
await cacheService.set('testString', 'test');
|
||||
await cacheService.set('testNumber1', 123);
|
||||
|
||||
await expect(cacheService.get<string>('testString')).resolves.toBe('test');
|
||||
expect(typeof (await cacheService.get<string>('testString'))).toBe('string');
|
||||
await expect(cacheService.get<number>('testNumber')).resolves.toBe(123);
|
||||
expect(typeof (await cacheService.get<number>('testNumber'))).toBe('number');
|
||||
await expect(cacheService.get('testString')).resolves.toBe('test');
|
||||
expect(typeof (await cacheService.get('testString'))).toBe('string');
|
||||
await expect(cacheService.get('testNumber1')).resolves.toBe(123);
|
||||
expect(typeof (await cacheService.get('testNumber1'))).toBe('number');
|
||||
});
|
||||
|
||||
test('should honour ttl values', async () => {
|
||||
// set default TTL to 10ms
|
||||
config.set('cache.memory.ttl', 10);
|
||||
|
||||
await cacheService.set<string>('testString', 'test');
|
||||
await cacheService.set<number>('testNumber', 123, 1000);
|
||||
await cacheService.set('testString', 'test');
|
||||
await cacheService.set('testNumber1', 123, 1000);
|
||||
|
||||
const store = (await cacheService.getCache())?.store;
|
||||
|
||||
expect(store).toBeDefined();
|
||||
|
||||
await expect(store!.ttl('testString')).resolves.toBeLessThanOrEqual(100);
|
||||
await expect(store!.ttl('testNumber')).resolves.toBeLessThanOrEqual(1000);
|
||||
await expect(store!.ttl('testNumber1')).resolves.toBeLessThanOrEqual(1000);
|
||||
|
||||
await expect(cacheService.get<string>('testString')).resolves.toBe('test');
|
||||
await expect(cacheService.get<number>('testNumber')).resolves.toBe(123);
|
||||
await expect(cacheService.get('testString')).resolves.toBe('test');
|
||||
await expect(cacheService.get('testNumber1')).resolves.toBe(123);
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
|
||||
await expect(cacheService.get<string>('testString')).resolves.toBeUndefined();
|
||||
await expect(cacheService.get<number>('testNumber')).resolves.toBe(123);
|
||||
await expect(cacheService.get('testString')).resolves.toBeUndefined();
|
||||
await expect(cacheService.get('testNumber1')).resolves.toBe(123);
|
||||
});
|
||||
|
||||
test('should set and remove values', async () => {
|
||||
await cacheService.set<string>('testString', 'test');
|
||||
await expect(cacheService.get<string>('testString')).resolves.toBe('test');
|
||||
await cacheService.set('testString', 'test');
|
||||
await expect(cacheService.get('testString')).resolves.toBe('test');
|
||||
await cacheService.delete('testString');
|
||||
await expect(cacheService.get<string>('testString')).resolves.toBeUndefined();
|
||||
await expect(cacheService.get('testString')).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
test('should calculate maxSize', async () => {
|
||||
@@ -73,65 +114,228 @@ describe('cacheService', () => {
|
||||
await cacheService.destroy();
|
||||
|
||||
// 16 bytes because stringify wraps the string in quotes, so 2 bytes for the quotes
|
||||
await cacheService.set<string>('testString', 'withoutUnicode');
|
||||
await expect(cacheService.get<string>('testString')).resolves.toBe('withoutUnicode');
|
||||
await cacheService.set('testString', 'withoutUnicode');
|
||||
await expect(cacheService.get('testString')).resolves.toBe('withoutUnicode');
|
||||
|
||||
await cacheService.destroy();
|
||||
|
||||
// should not fit!
|
||||
await cacheService.set<string>('testString', 'withUnicodeԱԲԳ');
|
||||
await expect(cacheService.get<string>('testString')).resolves.toBeUndefined();
|
||||
await cacheService.set('testString', 'withUnicodeԱԲԳ');
|
||||
await expect(cacheService.get('testString')).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
test('should set and get complex objects', async () => {
|
||||
interface TestObject {
|
||||
test: string;
|
||||
test2: number;
|
||||
test3?: TestObject & { test4: TestObject };
|
||||
}
|
||||
|
||||
const testObject: TestObject = {
|
||||
test: 'test',
|
||||
test2: 123,
|
||||
test3: {
|
||||
test: 'test3',
|
||||
test2: 123,
|
||||
test4: {
|
||||
test: 'test4',
|
||||
test2: 123,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
await cacheService.set<TestObject>('testObject', testObject);
|
||||
await expect(cacheService.get<TestObject>('testObject')).resolves.toMatchObject(testObject);
|
||||
await cacheService.set('testObject', testObject);
|
||||
await expect(cacheService.get('testObject')).resolves.toMatchObject(testObject);
|
||||
});
|
||||
|
||||
test('should set and get multiple values', async () => {
|
||||
config.set('executions.mode', 'regular');
|
||||
config.set('cache.backend', 'auto');
|
||||
await cacheService.destroy();
|
||||
expect(cacheService.isRedisCache()).toBe(false);
|
||||
|
||||
await cacheService.setMany<string>([
|
||||
await cacheService.setMany([
|
||||
['testString', 'test'],
|
||||
['testString2', 'test2'],
|
||||
]);
|
||||
await cacheService.setMany<number>([
|
||||
['testNumber', 123],
|
||||
await cacheService.setMany([
|
||||
['testNumber1', 123],
|
||||
['testNumber2', 456],
|
||||
]);
|
||||
await expect(cacheService.getMany(['testString', 'testString2'])).resolves.toStrictEqual([
|
||||
'test',
|
||||
'test2',
|
||||
]);
|
||||
await expect(cacheService.getMany(['testNumber1', 'testNumber2'])).resolves.toStrictEqual([
|
||||
123, 456,
|
||||
]);
|
||||
});
|
||||
|
||||
test('should create a redis in queue mode', async () => {
|
||||
config.set('cache.backend', 'auto');
|
||||
config.set('executions.mode', 'queue');
|
||||
await cacheService.destroy();
|
||||
await cacheService.init();
|
||||
|
||||
const cache = await cacheService.getCache();
|
||||
await expect(cacheService.getCache()).resolves.toBeDefined();
|
||||
const candidate = (await cacheService.getCache()) as RedisCache;
|
||||
expect(candidate.store.client).toBeDefined();
|
||||
});
|
||||
|
||||
test('should create a redis cache if asked', async () => {
|
||||
config.set('cache.backend', 'redis');
|
||||
config.set('executions.mode', 'queue');
|
||||
await cacheService.destroy();
|
||||
await cacheService.init();
|
||||
|
||||
const cache = await cacheService.getCache();
|
||||
await expect(cacheService.getCache()).resolves.toBeDefined();
|
||||
const candidate = (await cacheService.getCache()) as RedisCache;
|
||||
expect(candidate.store.client).toBeDefined();
|
||||
});
|
||||
|
||||
test('should get/set/delete redis cache', async () => {
|
||||
config.set('cache.backend', 'redis');
|
||||
config.set('executions.mode', 'queue');
|
||||
await cacheService.destroy();
|
||||
await cacheService.init();
|
||||
|
||||
await cacheService.set('testObject', testObject);
|
||||
await expect(cacheService.get('testObject')).resolves.toMatchObject(testObject);
|
||||
await cacheService.delete('testObject');
|
||||
await expect(cacheService.get('testObject')).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
// NOTE: mset and mget are not supported by ioredis-mock
|
||||
// test('should set and get multiple values with redis', async () => {
|
||||
// });
|
||||
|
||||
test('should return fallback value if key is not set', async () => {
|
||||
await cacheService.reset();
|
||||
await expect(cacheService.get('testString')).resolves.toBeUndefined();
|
||||
await expect(
|
||||
cacheService.get('testString', {
|
||||
fallbackValue: 'fallback',
|
||||
}),
|
||||
).resolves.toBe('fallback');
|
||||
});
|
||||
|
||||
test('should call refreshFunction if key is not set', async () => {
|
||||
await cacheService.reset();
|
||||
await expect(cacheService.get('testString')).resolves.toBeUndefined();
|
||||
await expect(
|
||||
cacheService.get('testString', {
|
||||
refreshFunction: async () => 'refreshed',
|
||||
fallbackValue: 'this should not be returned',
|
||||
}),
|
||||
).resolves.toBe('refreshed');
|
||||
});
|
||||
|
||||
test('should transparently handle disabled cache', async () => {
|
||||
await cacheService.disable();
|
||||
await expect(cacheService.get('testString')).resolves.toBeUndefined();
|
||||
await cacheService.set('testString', 'whatever');
|
||||
await expect(cacheService.get('testString')).resolves.toBeUndefined();
|
||||
await expect(
|
||||
cacheService.get('testString', {
|
||||
fallbackValue: 'fallback',
|
||||
}),
|
||||
).resolves.toBe('fallback');
|
||||
await expect(
|
||||
cacheService.get('testString', {
|
||||
refreshFunction: async () => 'refreshed',
|
||||
fallbackValue: 'this should not be returned',
|
||||
}),
|
||||
).resolves.toBe('refreshed');
|
||||
});
|
||||
|
||||
test('should set and get partial results', async () => {
|
||||
await cacheService.setMany([
|
||||
['testNumber1', 123],
|
||||
['testNumber2', 456],
|
||||
]);
|
||||
await expect(cacheService.getMany(['testNumber1', 'testNumber2'])).resolves.toStrictEqual([
|
||||
123, 456,
|
||||
]);
|
||||
await expect(cacheService.getMany(['testNumber3', 'testNumber2'])).resolves.toStrictEqual([
|
||||
undefined,
|
||||
456,
|
||||
]);
|
||||
});
|
||||
|
||||
test('should getMany and fix partial results and set single key', async () => {
|
||||
await cacheService.setMany([
|
||||
['testNumber1', 123],
|
||||
['testNumber2', 456],
|
||||
]);
|
||||
await expect(
|
||||
cacheService.getMany<string>(['testString', 'testString2']),
|
||||
).resolves.toStrictEqual(['test', 'test2']);
|
||||
cacheService.getMany(['testNumber1', 'testNumber2', 'testNumber3']),
|
||||
).resolves.toStrictEqual([123, 456, undefined]);
|
||||
await expect(cacheService.get('testNumber3')).resolves.toBeUndefined();
|
||||
await expect(
|
||||
cacheService.getMany<number>(['testNumber', 'testNumber2']),
|
||||
cacheService.getMany(['testNumber1', 'testNumber2', 'testNumber3'], {
|
||||
async refreshFunctionEach(key) {
|
||||
return key === 'testNumber3' ? 789 : undefined;
|
||||
},
|
||||
}),
|
||||
).resolves.toStrictEqual([123, 456, 789]);
|
||||
await expect(cacheService.get('testNumber3')).resolves.toBe(789);
|
||||
});
|
||||
|
||||
test('should getMany and set all keys', async () => {
|
||||
await cacheService.setMany([
|
||||
['testNumber1', 123],
|
||||
['testNumber2', 456],
|
||||
]);
|
||||
await expect(
|
||||
cacheService.getMany(['testNumber1', 'testNumber2', 'testNumber3']),
|
||||
).resolves.toStrictEqual([123, 456, undefined]);
|
||||
await expect(cacheService.get('testNumber3')).resolves.toBeUndefined();
|
||||
await expect(
|
||||
cacheService.getMany(['testNumber1', 'testNumber2', 'testNumber3'], {
|
||||
async refreshFunctionMany(keys) {
|
||||
return [111, 222, 333];
|
||||
},
|
||||
}),
|
||||
).resolves.toStrictEqual([111, 222, 333]);
|
||||
await expect(cacheService.get('testNumber1')).resolves.toBe(111);
|
||||
await expect(cacheService.get('testNumber2')).resolves.toBe(222);
|
||||
await expect(cacheService.get('testNumber3')).resolves.toBe(333);
|
||||
});
|
||||
|
||||
test('should set and get multiple values with fallbackValue', async () => {
|
||||
await cacheService.disable();
|
||||
await cacheService.setMany([
|
||||
['testNumber1', 123],
|
||||
['testNumber2', 456],
|
||||
]);
|
||||
await expect(cacheService.getMany(['testNumber1', 'testNumber2'])).resolves.toStrictEqual([
|
||||
undefined,
|
||||
undefined,
|
||||
]);
|
||||
await expect(
|
||||
cacheService.getMany(['testNumber1', 'testNumber2'], {
|
||||
fallbackValues: [123, 456],
|
||||
}),
|
||||
).resolves.toStrictEqual([123, 456]);
|
||||
await expect(
|
||||
cacheService.getMany(['testNumber1', 'testNumber2'], {
|
||||
refreshFunctionMany: async () => [123, 456],
|
||||
fallbackValues: [0, 1],
|
||||
}),
|
||||
).resolves.toStrictEqual([123, 456]);
|
||||
});
|
||||
// This test is skipped because it requires the Redis service
|
||||
// test('should create a redis cache if asked', async () => {
|
||||
// config.set('cache.backend', 'redis');
|
||||
// await cacheService.init();
|
||||
// expect(cacheService.getCacheInstance()).toBeDefined();
|
||||
// const candidate = cacheService.getCacheInstance() as RedisCache;
|
||||
// expect(candidate.store.client).toBeDefined();
|
||||
// });
|
||||
|
||||
test('should deal with unicode keys', async () => {
|
||||
const key = '? > ":< ! withUnicodeԱԲԳ';
|
||||
await cacheService.set(key, 'test');
|
||||
await expect(cacheService.get(key)).resolves.toBe('test');
|
||||
await cacheService.delete(key);
|
||||
await expect(cacheService.get(key)).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
test('should deal with unicode keys in redis', async () => {
|
||||
config.set('cache.backend', 'redis');
|
||||
config.set('executions.mode', 'queue');
|
||||
await cacheService.destroy();
|
||||
await cacheService.init();
|
||||
const key = '? > ":< ! withUnicodeԱԲԳ';
|
||||
|
||||
expect(((await cacheService.getCache()) as RedisCache).store.client).toBeDefined();
|
||||
|
||||
await cacheService.set(key, 'test');
|
||||
await expect(cacheService.get(key)).resolves.toBe('test');
|
||||
await cacheService.delete(key);
|
||||
await expect(cacheService.get(key)).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
test('should not cache null or undefined values', async () => {
|
||||
await cacheService.set('nullValue', null);
|
||||
await cacheService.set('undefValue', undefined);
|
||||
await cacheService.set('normalValue', 'test');
|
||||
|
||||
await expect(cacheService.get('normalValue')).resolves.toBe('test');
|
||||
await expect(cacheService.get('undefValue')).resolves.toBeUndefined();
|
||||
await expect(cacheService.get('nullValue')).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
138
packages/cli/test/unit/services/redis.service.test.ts
Normal file
138
packages/cli/test/unit/services/redis.service.test.ts
Normal file
@@ -0,0 +1,138 @@
|
||||
import Container from 'typedi';
|
||||
import config from '@/config';
|
||||
import { LoggerProxy } from 'n8n-workflow';
|
||||
import { getLogger } from '@/Logger';
|
||||
import { RedisService } from '@/services/redis.service';
|
||||
|
||||
const redisService = Container.get(RedisService);
|
||||
|
||||
function setDefaultConfig() {
|
||||
config.set('executions.mode', 'queue');
|
||||
}
|
||||
|
||||
interface TestObject {
|
||||
test: string;
|
||||
test2: number;
|
||||
test3?: TestObject & { test4: TestObject };
|
||||
}
|
||||
|
||||
const testObject: TestObject = {
|
||||
test: 'test',
|
||||
test2: 123,
|
||||
test3: {
|
||||
test: 'test3',
|
||||
test2: 123,
|
||||
test4: {
|
||||
test: 'test4',
|
||||
test2: 123,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const PUBSUB_CHANNEL = 'testchannel';
|
||||
const LIST_CHANNEL = 'testlist';
|
||||
const STREAM_CHANNEL = 'teststream';
|
||||
|
||||
describe('cacheService', () => {
|
||||
beforeAll(async () => {
|
||||
LoggerProxy.init(getLogger());
|
||||
jest.mock('ioredis', () => {
|
||||
const Redis = require('ioredis-mock');
|
||||
if (typeof Redis === 'object') {
|
||||
// the first mock is an ioredis shim because ioredis-mock depends on it
|
||||
// https://github.com/stipsan/ioredis-mock/blob/master/src/index.js#L101-L111
|
||||
return {
|
||||
Command: { _transformer: { argument: {}, reply: {} } },
|
||||
};
|
||||
}
|
||||
// second mock for our code
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return function (...args: any) {
|
||||
return new Redis(args);
|
||||
};
|
||||
});
|
||||
setDefaultConfig();
|
||||
});
|
||||
|
||||
test('should create pubsub publisher and subscriber with handler', async () => {
|
||||
const pub = await redisService.getPubSubPublisher();
|
||||
const sub = await redisService.getPubSubSubscriber();
|
||||
expect(pub).toBeDefined();
|
||||
expect(sub).toBeDefined();
|
||||
|
||||
const mockHandler = jest.fn();
|
||||
mockHandler.mockImplementation((channel: string, message: string) => {});
|
||||
sub.addMessageHandler(PUBSUB_CHANNEL, mockHandler);
|
||||
await sub.subscribe(PUBSUB_CHANNEL);
|
||||
await pub.publish(PUBSUB_CHANNEL, 'test');
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(async () => {
|
||||
resolve(0);
|
||||
}, 50),
|
||||
);
|
||||
expect(mockHandler).toHaveBeenCalled();
|
||||
await sub.destroy();
|
||||
await pub.destroy();
|
||||
});
|
||||
|
||||
test('should create list sender and receiver', async () => {
|
||||
const sender = await redisService.getListSender();
|
||||
const receiver = await redisService.getListReceiver();
|
||||
expect(sender).toBeDefined();
|
||||
expect(receiver).toBeDefined();
|
||||
await sender.prepend(LIST_CHANNEL, 'middle');
|
||||
await sender.prepend(LIST_CHANNEL, 'first');
|
||||
await sender.append(LIST_CHANNEL, 'end');
|
||||
let popResult = await receiver.popFromHead(LIST_CHANNEL);
|
||||
expect(popResult).toBe('first');
|
||||
popResult = await receiver.popFromTail(LIST_CHANNEL);
|
||||
expect(popResult).toBe('end');
|
||||
await sender.prepend(LIST_CHANNEL, 'somevalue');
|
||||
popResult = await receiver.popFromTail(LIST_CHANNEL);
|
||||
expect(popResult).toBe('middle');
|
||||
await sender.destroy();
|
||||
await receiver.destroy();
|
||||
});
|
||||
|
||||
// NOTE: This test is failing because the mock Redis client does not support streams apparently
|
||||
// eslint-disable-next-line n8n-local-rules/no-skipped-tests
|
||||
test.skip('should create stream producer and consumer', async () => {
|
||||
const consumer = await redisService.getStreamConsumer();
|
||||
const producer = await redisService.getStreamProducer();
|
||||
|
||||
expect(consumer).toBeDefined();
|
||||
expect(producer).toBeDefined();
|
||||
|
||||
const mockHandler = jest.fn();
|
||||
mockHandler.mockImplementation((stream: string, id: string, message: string[]) => {
|
||||
console.log('Received message', stream, id, message);
|
||||
});
|
||||
consumer.addMessageHandler('some handler', mockHandler);
|
||||
|
||||
await consumer.setPollingInterval(STREAM_CHANNEL, 50);
|
||||
await consumer.listenToStream(STREAM_CHANNEL);
|
||||
|
||||
let timeout;
|
||||
await new Promise((resolve) => {
|
||||
timeout = setTimeout(async () => {
|
||||
await producer.add(STREAM_CHANNEL, ['message', 'testMessage', 'event', 'testEveny']);
|
||||
resolve(0);
|
||||
}, 50);
|
||||
});
|
||||
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(async () => {
|
||||
resolve(0);
|
||||
}, 100),
|
||||
);
|
||||
|
||||
clearInterval(timeout);
|
||||
|
||||
consumer.stopListeningToStream(STREAM_CHANNEL);
|
||||
|
||||
expect(mockHandler).toHaveBeenCalled();
|
||||
|
||||
await consumer.destroy();
|
||||
await producer.destroy();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user