mirror of
https://github.com/coollabsio/coolify.git
synced 2026-03-11 08:55:47 +00:00
1739c04d32
Add configurable `capacity_wait_timeout` to runner configs and use it in `ProvisionGithubRunnerJob` to re-dispatch jobs while at capacity, preserving initial wait start time and stopping after timeout. Improve runner provisioning by caching an extracted runner template and copying it into new runner directories. Update Livewire GitHub Runners UI to lazy-load accessible repositories via `wire:init`, track loading state, add timeout input, and remove preinstall binary action. Add/extend feature tests for capacity retry timeout behavior and deferred repository loading.
336 lines
13 KiB
PHP
336 lines
13 KiB
PHP
<?php
|
|
|
|
namespace App\Jobs;
|
|
|
|
use App\Enums\GithubRunnerStatus;
|
|
use App\Models\GithubApp;
|
|
use App\Models\GithubRunnerConfig;
|
|
use App\Models\GithubRunnerExecution;
|
|
use Illuminate\Bus\Queueable;
|
|
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
|
use Illuminate\Foundation\Bus\Dispatchable;
|
|
use Illuminate\Queue\InteractsWithQueue;
|
|
use Illuminate\Queue\SerializesModels;
|
|
use Illuminate\Support\Facades\Http;
|
|
use Visus\Cuid2\Cuid2;
|
|
|
|
class ProvisionGithubRunnerJob implements ShouldBeEncrypted, ShouldQueue
|
|
{
|
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
|
|
|
public $timeout = 300;
|
|
|
|
public $tries = 3;
|
|
|
|
public function backoff(): array
|
|
{
|
|
return [5, 15, 30];
|
|
}
|
|
|
|
public function __construct(
|
|
public int $githubAppId,
|
|
public array $workflowJobPayload,
|
|
public string $organizationLogin,
|
|
public int $repositoryId = 0,
|
|
public ?string $capacityWaitStartedAt = null,
|
|
) {
|
|
$this->onQueue('high');
|
|
}
|
|
|
|
public function handle(): void
|
|
{
|
|
$workflowJobId = data_get($this->workflowJobPayload, 'id');
|
|
|
|
// Idempotency: skip if already provisioning for this job
|
|
if (GithubRunnerExecution::where('workflow_job_id', $workflowJobId)->exists()) {
|
|
return;
|
|
}
|
|
|
|
$githubApp = GithubApp::find($this->githubAppId);
|
|
if (! $githubApp) {
|
|
return;
|
|
}
|
|
|
|
$requestedLabels = data_get($this->workflowJobPayload, 'labels', []);
|
|
|
|
// Step 1: find configs that match labels (ignoring capacity)
|
|
$matchingConfigs = $this->findMatchingConfigsIgnoringCapacity($githubApp, $requestedLabels);
|
|
|
|
if ($matchingConfigs->isEmpty()) {
|
|
// No configured server handles these labels at all — nothing to do
|
|
return;
|
|
}
|
|
|
|
// Step 2: filter to configs that have capacity
|
|
$config = $matchingConfigs
|
|
->filter(fn ($c) => $c->hasCapacity())
|
|
->sortBy(fn ($c) => $c->activeRunnerCount())
|
|
->first();
|
|
|
|
if (! $config) {
|
|
// All matching configs are at capacity — wait and retry
|
|
$timeoutMinutes = $matchingConfigs->first()->capacity_wait_timeout;
|
|
$waitStartedAt = $this->capacityWaitStartedAt
|
|
? \Carbon\Carbon::parse($this->capacityWaitStartedAt)
|
|
: now();
|
|
|
|
if (now()->diffInMinutes($waitStartedAt) >= $timeoutMinutes) {
|
|
// Gave up waiting — log and drop so GitHub eventually cancels the job
|
|
logger()->warning('ProvisionGithubRunnerJob: gave up waiting for capacity', [
|
|
'workflow_job_id' => $workflowJobId,
|
|
'labels' => $requestedLabels,
|
|
'timeout_minutes' => $timeoutMinutes,
|
|
]);
|
|
|
|
return;
|
|
}
|
|
|
|
// Dispatch a new job in 15 seconds carrying the wait start timestamp
|
|
static::dispatch(
|
|
githubAppId: $this->githubAppId,
|
|
workflowJobPayload: $this->workflowJobPayload,
|
|
organizationLogin: $this->organizationLogin,
|
|
repositoryId: $this->repositoryId,
|
|
capacityWaitStartedAt: $this->capacityWaitStartedAt ?? now()->toIso8601String(),
|
|
)->delay(15);
|
|
|
|
return;
|
|
}
|
|
|
|
$runnerName = 'coolify-'.((string) new Cuid2(7));
|
|
$runnerDir = "{$config->runner_base_dir}/{$runnerName}";
|
|
|
|
$execution = GithubRunnerExecution::create([
|
|
'server_id' => $config->server_id,
|
|
'github_runner_config_id' => $config->id,
|
|
'status' => GithubRunnerStatus::Queued,
|
|
'runner_name' => $runnerName,
|
|
'runner_dir' => $runnerDir,
|
|
'workflow_job_id' => $workflowJobId,
|
|
'workflow_name' => data_get($this->workflowJobPayload, 'workflow_name'),
|
|
'repository_full_name' => data_get($this->workflowJobPayload, 'repository.full_name',
|
|
data_get($this->workflowJobPayload, 'head_repository.full_name')
|
|
),
|
|
]);
|
|
|
|
try {
|
|
$execution->update(['status' => GithubRunnerStatus::Provisioning]);
|
|
|
|
// Ensure a Coolify-managed runner group exists and the repo has access
|
|
$runnerGroupId = $this->ensureRunnerGroup($githubApp);
|
|
$this->ensureRepositoryInRunnerGroup($githubApp, $runnerGroupId);
|
|
|
|
// Generate JIT config via GitHub API
|
|
['encoded_jit_config' => $jitConfig, 'runner_id' => $runnerId] = $this->generateJitConfig($githubApp, $config, $runnerName, $requestedLabels, $runnerGroupId);
|
|
|
|
// Provision runner on server via SSH
|
|
$pid = $this->provisionRunner($config, $runnerName, $runnerDir, $jitConfig);
|
|
|
|
$execution->update([
|
|
'status' => GithubRunnerStatus::Running,
|
|
'pid' => $pid,
|
|
'runner_id' => $runnerId,
|
|
'started_at' => now(),
|
|
]);
|
|
} catch (\Throwable $e) {
|
|
$execution->update([
|
|
'status' => GithubRunnerStatus::Failed,
|
|
'error_message' => $e->getMessage(),
|
|
]);
|
|
|
|
// Attempt cleanup of the runner directory on the server
|
|
try {
|
|
$server = $config->server;
|
|
instant_remote_process(["rm -rf {$runnerDir}"], $server, throwError: false);
|
|
} catch (\Throwable) {
|
|
// Best-effort cleanup
|
|
}
|
|
|
|
throw $e;
|
|
}
|
|
}
|
|
|
|
private function findMatchingConfigsIgnoringCapacity(GithubApp $githubApp, array $requestedLabels): \Illuminate\Support\Collection
|
|
{
|
|
return GithubRunnerConfig::query()
|
|
->where('github_app_id', $githubApp->id)
|
|
->whereHas('githubApp', fn ($q) => $q->where('organization', $this->organizationLogin))
|
|
->where('is_enabled', true)
|
|
->with('server')
|
|
->get()
|
|
->filter(fn ($config) => $config->matchesLabels($requestedLabels))
|
|
->filter(fn ($config) => $config->server->isFunctional())
|
|
->values();
|
|
}
|
|
|
|
private function ensureRunnerGroup(GithubApp $githubApp): int
|
|
{
|
|
$token = generateGithubInstallationToken($githubApp);
|
|
$apiUrl = $githubApp->api_url ?? 'https://api.github.com';
|
|
|
|
if ($githubApp->runner_group_id) {
|
|
// Ensure existing group allows public repos
|
|
Http::withHeaders([
|
|
'Authorization' => "Bearer {$token}",
|
|
'Accept' => 'application/vnd.github+json',
|
|
'X-GitHub-Api-Version' => '2022-11-28',
|
|
])->patch("{$apiUrl}/orgs/{$githubApp->organization}/actions/runner-groups/{$githubApp->runner_group_id}", [
|
|
'allows_public_repositories' => true,
|
|
]);
|
|
|
|
return $githubApp->runner_group_id;
|
|
}
|
|
|
|
$groupName = 'Coolify-'.((string) new Cuid2(7));
|
|
|
|
$response = Http::withHeaders([
|
|
'Authorization' => "Bearer {$token}",
|
|
'Accept' => 'application/vnd.github+json',
|
|
'X-GitHub-Api-Version' => '2022-11-28',
|
|
])->post("{$apiUrl}/orgs/{$githubApp->organization}/actions/runner-groups", [
|
|
'name' => $groupName,
|
|
'visibility' => 'selected',
|
|
'allows_public_repositories' => true,
|
|
]);
|
|
|
|
if (! $response->successful()) {
|
|
throw new \RuntimeException(
|
|
'Failed to create runner group: '.data_get($response->json(), 'message', $response->body())
|
|
);
|
|
}
|
|
|
|
$runnerGroupId = (int) data_get($response->json(), 'id');
|
|
$githubApp->update(['runner_group_id' => $runnerGroupId]);
|
|
|
|
return $runnerGroupId;
|
|
}
|
|
|
|
private function ensureRepositoryInRunnerGroup(GithubApp $githubApp, int $runnerGroupId): void
|
|
{
|
|
if ($this->repositoryId <= 0) {
|
|
ray("Skipping runner group repo assignment — repositoryId is {$this->repositoryId}");
|
|
|
|
return;
|
|
}
|
|
|
|
$token = generateGithubInstallationToken($githubApp);
|
|
$apiUrl = $githubApp->api_url ?? 'https://api.github.com';
|
|
$url = "{$apiUrl}/orgs/{$githubApp->organization}/actions/runner-groups/{$runnerGroupId}/repositories/{$this->repositoryId}";
|
|
|
|
ray("Adding repository {$this->repositoryId} to runner group {$runnerGroupId}: PUT {$url}");
|
|
|
|
$response = Http::withHeaders([
|
|
'Authorization' => "Bearer {$token}",
|
|
'Accept' => 'application/vnd.github+json',
|
|
'X-GitHub-Api-Version' => '2022-11-28',
|
|
])->withBody('', 'application/json')->put($url);
|
|
|
|
if (! $response->successful()) {
|
|
ray('Failed to add repository to runner group: '.$response->status().' '.data_get($response->json(), 'message', $response->body()));
|
|
}
|
|
}
|
|
|
|
private function generateJitConfig(GithubApp $githubApp, GithubRunnerConfig $config, string $runnerName, array $requestedLabels, int $runnerGroupId): array
|
|
{
|
|
$token = generateGithubInstallationToken($githubApp);
|
|
$apiUrl = $githubApp->api_url ?? 'https://api.github.com';
|
|
|
|
$response = Http::withHeaders([
|
|
'Authorization' => "Bearer {$token}",
|
|
'Accept' => 'application/vnd.github+json',
|
|
'X-GitHub-Api-Version' => '2022-11-28',
|
|
])->post("{$apiUrl}/orgs/{$config->organization}/actions/runners/generate-jitconfig", [
|
|
'name' => $runnerName,
|
|
'runner_group_id' => $runnerGroupId,
|
|
'labels' => $requestedLabels,
|
|
'work_folder' => '_work',
|
|
]);
|
|
|
|
if (! $response->successful()) {
|
|
throw new \RuntimeException(
|
|
'Failed to generate JIT runner config: '.data_get($response->json(), 'message', $response->body())
|
|
);
|
|
}
|
|
|
|
return [
|
|
'encoded_jit_config' => data_get($response->json(), 'encoded_jit_config'),
|
|
'runner_id' => data_get($response->json(), 'runner.id'),
|
|
];
|
|
}
|
|
|
|
private function provisionRunner(GithubRunnerConfig $config, string $runnerName, string $runnerDir, string $jitConfig): int
|
|
{
|
|
$server = $config->server;
|
|
$user = $config->runner_user;
|
|
$baseDir = $config->runner_base_dir;
|
|
$cacheDir = "{$baseDir}/.cache";
|
|
// Detect architecture from server
|
|
$uname = trim(instant_remote_process(['uname -m'], $server));
|
|
$arch = $uname === 'aarch64' ? 'arm64' : 'x64';
|
|
|
|
$version = $config->runner_version ?? $this->getLatestRunnerVersion($config, $arch);
|
|
|
|
// Ensure runner user and directories exist
|
|
instant_remote_process([
|
|
"id -u {$user} &>/dev/null || useradd -m -s /bin/bash {$user}",
|
|
"usermod -aG docker {$user}",
|
|
"mkdir -p {$cacheDir}",
|
|
"mkdir -p {$runnerDir}",
|
|
], $server);
|
|
|
|
// Download runner binary if not cached, then populate runner dir from pre-extracted template
|
|
$tarball = "actions-runner-linux-{$arch}-{$version}.tar.gz";
|
|
$templateDir = "{$baseDir}/.templates/runner-{$arch}-{$version}";
|
|
instant_remote_process([
|
|
"if [ ! -f {$cacheDir}/{$tarball} ]; then curl -sL https://github.com/actions/runner/releases/download/v{$version}/{$tarball} -o {$cacheDir}/{$tarball}; fi",
|
|
"if [ ! -d {$templateDir} ]; then mkdir -p {$templateDir} && tar xzf {$cacheDir}/{$tarball} -C {$templateDir} && chown -R {$user}:{$user} {$templateDir}; fi",
|
|
"cp -r {$templateDir}/. {$runnerDir}",
|
|
"chown -R {$user}:{$user} {$runnerDir}",
|
|
], $server);
|
|
|
|
// Start the JIT runner in background
|
|
$output = instant_remote_process([
|
|
"cd {$runnerDir} && sudo -u {$user} nohup ./run.sh --jitconfig {$jitConfig} > {$runnerDir}/runner.log 2>&1 & echo \$!",
|
|
], $server);
|
|
|
|
$pid = (int) trim($output);
|
|
if ($pid <= 0) {
|
|
throw new \RuntimeException('Failed to start runner process — no PID returned.');
|
|
}
|
|
|
|
return $pid;
|
|
}
|
|
|
|
private function getLatestRunnerVersion(GithubRunnerConfig $config, string $arch): string
|
|
{
|
|
$githubApp = GithubApp::find($this->githubAppId);
|
|
$apiUrl = $githubApp?->api_url ?? 'https://api.github.com';
|
|
|
|
try {
|
|
$token = generateGithubInstallationToken($githubApp);
|
|
$response = Http::withHeaders([
|
|
'Authorization' => "Bearer {$token}",
|
|
'Accept' => 'application/vnd.github+json',
|
|
'X-GitHub-Api-Version' => '2022-11-28',
|
|
])->get("{$apiUrl}/orgs/{$config->organization}/actions/runners/downloads");
|
|
|
|
if ($response->successful()) {
|
|
$download = collect($response->json())
|
|
->first(fn ($d) => data_get($d, 'os') === 'linux' && data_get($d, 'architecture') === $arch);
|
|
|
|
if ($download) {
|
|
// Extract version from filename like "actions-runner-linux-x64-2.321.0.tar.gz"
|
|
preg_match('/(\d+\.\d+\.\d+)/', data_get($download, 'filename', ''), $matches);
|
|
if (! empty($matches[1])) {
|
|
return $matches[1];
|
|
}
|
|
}
|
|
}
|
|
} catch (\Throwable) {
|
|
// Fall through to default
|
|
}
|
|
|
|
return '2.321.0';
|
|
}
|
|
}
|