fix(core): Add recoveryInProgress flag file (#6962)
Issue: during startup, unfinished executions trigger a recovery process that, under certain circumstances, can in itself crash the instance (e.g. by running our of memory), resulting in an infinite recovery loop This PR aims to change this behaviour by writing a flag file when the recovery process starts, and removing it when it finishes. In the case of a crash, this flag will persist and upon the next attempt, the recovery will instead do the absolute minimal (marking executions as 'crashed'), without attempting any 'crashable' actions.
This commit is contained in:
committed by
GitHub
parent
4fc69b776c
commit
7b96820218
@@ -98,6 +98,22 @@ export class MessageEventBusLogWriter {
|
||||
}
|
||||
}
|
||||
|
||||
startRecoveryProcess() {
|
||||
if (this.worker) {
|
||||
this.worker.postMessage({ command: 'startRecoveryProcess', data: {} });
|
||||
}
|
||||
}
|
||||
|
||||
isRecoveryProcessRunning(): boolean {
|
||||
return existsSync(this.getRecoveryInProgressFileName());
|
||||
}
|
||||
|
||||
endRecoveryProcess() {
|
||||
if (this.worker) {
|
||||
this.worker.postMessage({ command: 'endRecoveryProcess', data: {} });
|
||||
}
|
||||
}
|
||||
|
||||
private async startThread() {
|
||||
if (this.worker) {
|
||||
await this.close();
|
||||
@@ -240,6 +256,10 @@ export class MessageEventBusLogWriter {
|
||||
}
|
||||
}
|
||||
|
||||
getRecoveryInProgressFileName(): string {
|
||||
return `${MessageEventBusLogWriter.options.logFullBasePath}.recoveryInProgress`;
|
||||
}
|
||||
|
||||
cleanAllLogs() {
|
||||
for (let i = 0; i <= MessageEventBusLogWriter.options.keepNumberOfFiles; i++) {
|
||||
if (existsSync(this.getLogFileName(i))) {
|
||||
|
||||
Reference in New Issue
Block a user