onQueue('high'); } public function handle(): void { ray("[cleanup] Starting cleanup for workflow_job_id {$this->workflowJobId}"); $execution = GithubRunnerExecution::where('workflow_job_id', $this->workflowJobId) ->with('config.githubApp') ->first(); if (! $execution) { ray("[cleanup] No execution found for workflow_job_id {$this->workflowJobId} — skipping"); return; } // Already cleaned up if (in_array($execution->status, [GithubRunnerStatus::Completed, GithubRunnerStatus::Failed])) { ray("[cleanup] Execution {$execution->id} already in {$execution->status->value} — skipping"); return; } ray("[cleanup] Execution {$execution->id} transitioning from {$execution->status->value} → cleaning"); $execution->update(['status' => GithubRunnerStatus::Cleaning]); try { $server = $execution->server; if (! $server || ! $server->isFunctional()) { ray("[cleanup] Server not functional for execution {$execution->id}; marking failed to release capacity"); $this->deregisterFromGithub($execution); $execution->update([ 'status' => GithubRunnerStatus::Failed, 'error_message' => 'Cleanup skipped: server is not functional.', 'completed_at' => now(), ]); return; } if ($execution->pid) { instant_remote_process([ "kill {$execution->pid} 2>/dev/null || true", ], $server, throwError: false); } if ($execution->runner_dir) { instant_remote_process([ "rm -rf {$execution->runner_dir}", ], $server, throwError: false); } $this->deregisterFromGithub($execution); $execution->update([ 'status' => GithubRunnerStatus::Completed, 'completed_at' => now(), ]); ray("[cleanup] Execution {$execution->id} completed successfully"); } catch (\Throwable $e) { ray("[cleanup] Execution {$execution->id} cleanup failed: {$e->getMessage()}"); $execution->update([ 'status' => GithubRunnerStatus::Failed, 'error_message' => 'Cleanup failed: '.$e->getMessage(), 'completed_at' => now(), ]); } } public function failed(?\Throwable $exception): void { GithubRunnerExecution::query() ->where('workflow_job_id', $this->workflowJobId) ->whereIn('status', [ GithubRunnerStatus::Queued, GithubRunnerStatus::Provisioning, GithubRunnerStatus::Running, GithubRunnerStatus::Cleaning, ]) ->update([ 'status' => GithubRunnerStatus::Failed, 'error_message' => 'Cleanup failed: '.($exception?->getMessage() ?? 'unknown error'), 'completed_at' => now(), ]); } private function deregisterFromGithub(GithubRunnerExecution $execution): void { if (! $execution->runner_id) { ray('Runner deregister skipped: no runner_id for execution '.$execution->id); return; } $githubApp = $execution->config?->githubApp; if (! $githubApp || $githubApp->is_public) { ray('Runner deregister skipped: no githubApp or is_public for execution '.$execution->id); return; } $org = $githubApp->organization; if (! $org) { ray('Runner deregister skipped: no organization for execution '.$execution->id); return; } try { $token = generateGithubInstallationToken($githubApp); $apiUrl = $githubApp->api_url ?? 'https://api.github.com'; ray("Deregistering runner {$execution->runner_id} from {$org} via DELETE /orgs/{$org}/actions/runners/{$execution->runner_id}"); $response = Http::GitHub($apiUrl, $token) ->delete("/orgs/{$org}/actions/runners/{$execution->runner_id}"); ray('Runner deregister response: '.$response->status().' '.$response->body()); } catch (\Throwable $e) { ray('Runner deregister failed: '.$e->getMessage()); } } }