fix(core): Add recoveryInProgress flag file (#6962)

Issue: during startup, unfinished executions trigger a recovery process
that, under certain circumstances, can in itself crash the instance
(e.g. by running our of memory), resulting in an infinite recovery loop

This PR aims to change this behaviour by writing a flag file when the
recovery process starts, and removing it when it finishes. In the case
of a crash, this flag will persist and upon the next attempt, the
recovery will instead do the absolute minimal (marking executions as
'crashed'), without attempting any 'crashable' actions.
This commit is contained in:
Michael Auerswald
2023-08-18 17:12:24 +02:00
committed by GitHub
parent 4fc69b776c
commit 7b96820218
3 changed files with 93 additions and 7 deletions

View File

@@ -98,6 +98,22 @@ export class MessageEventBusLogWriter {
}
}
startRecoveryProcess() {
if (this.worker) {
this.worker.postMessage({ command: 'startRecoveryProcess', data: {} });
}
}
isRecoveryProcessRunning(): boolean {
return existsSync(this.getRecoveryInProgressFileName());
}
endRecoveryProcess() {
if (this.worker) {
this.worker.postMessage({ command: 'endRecoveryProcess', data: {} });
}
}
private async startThread() {
if (this.worker) {
await this.close();
@@ -240,6 +256,10 @@ export class MessageEventBusLogWriter {
}
}
getRecoveryInProgressFileName(): string {
return `${MessageEventBusLogWriter.options.logFullBasePath}.recoveryInProgress`;
}
cleanAllLogs() {
for (let i = 0; i <= MessageEventBusLogWriter.options.keepNumberOfFiles; i++) {
if (existsSync(this.getLogFileName(i))) {