onQueue('high'); } public function handle(): void { $server = $this->database->destination->server; $containerName = $this->database->uuid; $stanza = PgBackrestService::getStanzaName($this->database); $backupTimestamp = time(); try { $this->restore->updateStatus('running', 'Starting PgBackRest restore.'); $this->preflight($stanza); $this->restore->appendLog('Stopping PostgreSQL container.'); StopDatabase::run($this->database, dockerCleanup: false); $this->restore->appendLog('Backing up current PGDATA before restore.'); $backupPath = $this->backupCurrentPgData($backupTimestamp); try { $this->restore->appendLog('Clearing PGDATA directory.'); $this->clearPgData(); $this->restore->appendLog('Restoring via PgBackRest sidecar container.'); $this->runRestoreSidecar($stanza); $this->restore->appendLog('Verifying restored database.'); $this->verifyRestore(); $this->restore->appendLog('Starting PostgreSQL after restore.'); StartPostgresql::run($this->database); // Restore succeeded, safely remove the backup $this->restore->appendLog('Removing backup of previous database.'); $this->removePgDataBackup($backupPath); $this->restore->updateStatus('success', 'PgBackRest restore completed successfully.'); $team = $this->database->team(); if ($team) { $team->notify(new DatabaseRestoreSuccess( $this->database, $this->restore->target_label, $this->targetTime )); } } catch (Throwable $restoreError) { // Restore failed, attempt to recover $this->restore->appendLog('Restore failed, attempting to recover from backup: '.$restoreError->getMessage()); $this->recoverFromBackup($backupPath); throw $restoreError; } } catch (Throwable $e) { Log::error('PgBackRest restore failed', [ 'database' => $this->database->uuid, 'restore_id' => $this->restore->uuid, 'error' => $e->getMessage(), 'trace' => $e->getTraceAsString(), ]); $this->restore->updateStatus('failed', 'Restore failed: '.$e->getMessage()); $team = $this->database->team(); if ($team) { $team->notify(new DatabaseRestoreFailed( $this->database, $e->getMessage(), $this->restore->target_label )); } try { $this->restore->appendLog('Attempting to restart PostgreSQL after failed restore.'); StartPostgresql::run($this->database); } catch (Throwable $restartError) { $this->restore->appendLog('Failed to restart PostgreSQL: '.$restartError->getMessage()); } throw $e; } } private function preflight(string $stanza): void { $this->restore->appendLog('Running pre-flight validation.'); if (! $this->database->hasPgBackrestBackups()) { throw new \RuntimeException('No PgBackRest backup configuration found for this database.'); } $backup = $this->database->pgbackrestBackups()->where('enabled', true)->first(); if (! $backup) { throw new \RuntimeException('No enabled PgBackRest backup configuration found.'); } $s3Repos = $backup->pgbackrestRepos()->where('type', 's3')->where('enabled', true)->get(); foreach ($s3Repos as $s3Repo) { $s3 = $s3Repo->s3Storage; if (! $s3) { throw new \RuntimeException("S3 storage configuration not found for repo {$s3Repo->repo_number}."); } try { $s3->testConnection(shouldSave: true); } catch (Throwable $e) { throw new \RuntimeException("S3 connection test failed for repo {$s3Repo->repo_number}: ".$e->getMessage()); } } $pgdataVolume = $this->database->pgdataVolume(); if (! $pgdataVolume) { throw new \RuntimeException('PGDATA volume not found.'); } $repoVolume = $this->database->pgbackrestRepoVolume(); $hasLocalRepo = $backup->hasLocalRepo(); if (! $repoVolume && $hasLocalRepo) { throw new \RuntimeException('PgBackRest repository volume not found.'); } $this->restore->appendLog('Verifying backup exists in repository via sidecar container.'); $info = $this->runInfoSidecar($stanza, $backup); if (! PgBackrestService::stanzaExists($info)) { throw new \RuntimeException('PgBackRest stanza does not exist or is not healthy.'); } if (! PgBackrestService::hasBackups($info)) { throw new \RuntimeException('No backups found in PgBackRest repository.'); } if ($this->execution && $this->execution->pgbackrest_label) { $targetBackup = PgBackrestService::findBackupByLabel($info, $this->execution->pgbackrest_label); if (! $targetBackup) { throw new \RuntimeException("Backup with label '{$this->execution->pgbackrest_label}' not found in repository."); } $this->restore->appendLog("Target backup verified: {$this->execution->pgbackrest_label}"); } else { $latestBackup = PgBackrestService::getLatestBackup($info); if ($latestBackup) { $this->restore->appendLog("Will restore from latest backup: {$latestBackup['label']}"); } } $this->restore->appendLog('Pre-flight validation passed.'); } private function runInfoSidecar(string $stanza, $backup): array { $server = $this->database->destination->server; $configDir = $this->getHostPath($this->database->workdir().'/pgbackrest'); $network = $this->database->destination->network; $mounts = []; $envPieces = []; $repoVolume = $this->database->pgbackrestRepoVolume(); if ($repoVolume) { $repoMount = $repoVolume->host_path ?: $repoVolume->name; $mounts[] = "-v {$repoMount}:/var/lib/pgbackrest"; } $mounts[] = "-v {$configDir}:/etc/pgbackrest:ro"; $s3EnvVars = PgBackrestService::buildS3EnvVars($backup); foreach ($s3EnvVars as $key => $value) { $escapedValue = addslashes($value); $envPieces[] = "-e {$key}=\"{$escapedValue}\""; } $infoCmd = PgBackrestService::buildInfoCommand($stanza, true); $sidecarName = 'pgbackrest-info-'.$this->database->uuid.'-'.time(); $cmd = sprintf( 'docker run --rm --name %s --network %s %s %s %s sh -c \'%s\' 2>&1', $sidecarName, $network, implode(' ', $envPieces), implode(' ', $mounts), $this->getSidecarImage(), $this->getInstallAndRunCommand($infoCmd) ); $output = instant_remote_process([$cmd], $server, false, false, 120, disableMultiplexing: true); if ($output === null || $output === '') { throw new \RuntimeException('Failed to get PgBackRest info - command returned no output. Command: '.$cmd); } $jsonStart = strpos($output, '['); if ($jsonStart !== false) { $output = substr($output, $jsonStart); } $info = PgBackrestService::parseInfoJson($output); if ($info === null) { throw new \RuntimeException('Failed to parse PgBackRest info output: '.$output); } return $info; } private function backupCurrentPgData(int $timestamp): string { $server = $this->database->destination->server; $pgdataVolume = $this->database->pgdataVolume(); if (! $pgdataVolume) { throw new \RuntimeException('PGDATA volume not found.'); } $mount = $pgdataVolume->host_path ?: $pgdataVolume->name; $backupPath = "{$mount}_backup_{$timestamp}"; $backupCmd = "docker run --rm -v {$mount}:/data -v {$mount}_backup_{$timestamp}:/backup alpine sh -c 'cp -a /data/* /backup/ 2>/dev/null || true'"; instant_remote_process([$backupCmd], $server, false, false, 600, disableMultiplexing: true); $this->restore->appendLog("PGDATA backed up to temporary location: {$backupPath}"); return $backupPath; } private function recoverFromBackup(string $backupPath): void { try { $server = $this->database->destination->server; $pgdataVolume = $this->database->pgdataVolume(); if (! $pgdataVolume) { $this->restore->appendLog('Warning: PGDATA volume not found, cannot recover from backup.'); return; } $mount = $pgdataVolume->host_path ?: $pgdataVolume->name; $this->restore->appendLog('Recovering PGDATA from backup...'); $recoverCmd = "docker run --rm -v {$mount}:/data -v {$backupPath}:/backup alpine sh -c 'rm -rf /data/* /data/.[!.]* /data/..?* 2>/dev/null || true && cp -a /backup/* /data/ 2>/dev/null || true'"; instant_remote_process([$recoverCmd], $server, false, false, 600, disableMultiplexing: true); $this->restore->appendLog('PGDATA recovered from backup.'); } catch (Throwable $e) { $this->restore->appendLog('Failed to recover from backup: '.$e->getMessage()); } } private function removePgDataBackup(string $backupPath): void { try { $server = $this->database->destination->server; $rmCmd = "docker run --rm -v {$backupPath}:/backup alpine sh -c 'rm -rf /backup/* /backup/.[!.]* /backup/..?* 2>/dev/null || true'"; instant_remote_process([$rmCmd], $server, false, false, 300, disableMultiplexing: true); $this->restore->appendLog('Temporary backup removed.'); } catch (Throwable $e) { $this->restore->appendLog('Warning: Failed to remove temporary backup: '.$e->getMessage()); } } private function clearPgData(): void { $server = $this->database->destination->server; $pgdataVolume = $this->database->pgdataVolume(); if (! $pgdataVolume) { throw new \RuntimeException('PGDATA volume not found.'); } $mount = $pgdataVolume->host_path ?: $pgdataVolume->name; $rmCmd = "docker run --rm -v {$mount}:/data alpine sh -c 'rm -rf /data/* /data/.[!.]* /data/..?* 2>/dev/null || true'"; instant_remote_process([$rmCmd], $server, false, false, 300, disableMultiplexing: true); $this->restore->appendLog('PGDATA directory cleared.'); } private function verifyRestore(): void { try { $pgdataVolume = $this->database->pgdataVolume(); if (! $pgdataVolume) { throw new \RuntimeException('PGDATA volume not found.'); } $server = $this->database->destination->server; $mount = $pgdataVolume->host_path ?: $pgdataVolume->name; // Verify PG_VERSION exists in restored data $checkCmd = "docker run --rm -v {$mount}:/data alpine test -f /data/PG_VERSION && echo 'OK' || echo 'FAIL'"; $result = instant_remote_process([$checkCmd], $server, false, false, 30, disableMultiplexing: true); if (trim($result) !== 'OK') { throw new \RuntimeException('Restored PGDATA does not contain valid PostgreSQL data (PG_VERSION not found).'); } $this->restore->appendLog('Restored database verified successfully.'); } catch (Throwable $e) { throw new \RuntimeException('Database verification failed: '.$e->getMessage()); } } private function runRestoreSidecar(string $stanza): void { $server = $this->database->destination->server; $configDir = $this->getHostPath($this->database->workdir().'/pgbackrest'); $network = $this->database->destination->network; $backup = $this->database->pgbackrestBackups()->where('enabled', true)->first(); if (! $backup) { throw new \RuntimeException('No enabled PgBackRest backup configuration found.'); } $pgdataVolume = $this->database->pgdataVolume(); $repoVolume = $this->database->pgbackrestRepoVolume(); $pgdataMount = $pgdataVolume->host_path ?: $pgdataVolume->name; $mounts = []; $mounts[] = "-v {$pgdataMount}:".PgBackrestService::PGDATA_PATH; if ($repoVolume) { $repoMount = $repoVolume->host_path ?: $repoVolume->name; $mounts[] = "-v {$repoMount}:/var/lib/pgbackrest"; } $mounts[] = "-v {$configDir}:/etc/pgbackrest:ro"; $envPieces = ['-e PGBACKREST_PG1_PATH='.PgBackrestService::PGDATA_PATH]; $s3EnvVars = PgBackrestService::buildS3EnvVars($backup); foreach ($s3EnvVars as $key => $value) { $escapedValue = addslashes($value); $envPieces[] = "-e {$key}=\"{$escapedValue}\""; } $restoreCmd = PgBackrestService::buildRestoreCommand( $stanza, $this->execution?->pgbackrest_label, $this->targetTime, 'info' ); $sidecarName = 'pgbackrest-restore-'.$this->database->uuid.'-'.time(); $fullRestoreScript = $this->getInstallAndRunCommand($restoreCmd).' && chown -R 999:999 '.PgBackrestService::PGDATA_PATH; $cmd = sprintf( 'docker run --rm --name %s --network %s %s %s %s sh -c \'%s\' 2>&1', $sidecarName, $network, implode(' ', $envPieces), implode(' ', $mounts), $this->getSidecarImage(), $fullRestoreScript ); $output = instant_remote_process([$cmd], $server, true, false, $this->timeout, disableMultiplexing: true); $this->restore->appendLog('Restore output: '.substr($output, 0, 2000)); } private function getSidecarImage(): string { return $this->database->image; } private function getInstallAndRunCommand(string $command): string { return PgBackrestService::buildInstallAndSetupCommand($command); } /** * Convert a path from the SSH target perspective to the Docker host perspective. */ private function getHostPath(string $path): string { return convertPathToDockerHost($path); } public function failed(?Throwable $exception): void { Log::channel('scheduled-errors')->error('PgBackRest restore permanently failed', [ 'job' => 'PgBackrestRestoreJob', 'database' => $this->database->uuid, 'restore_id' => $this->restore->uuid, 'error' => $exception?->getMessage(), 'trace' => $exception?->getTraceAsString(), ]); $this->restore->updateStatus('failed', 'Restore job permanently failed: '.($exception?->getMessage() ?? 'Unknown error')); $team = $this->database->team(); if ($team) { $team->notify(new DatabaseRestoreFailed( $this->database, $exception?->getMessage() ?? 'Unknown error', $this->restore->target_label )); } } }