mirror of
https://github.com/coollabsio/coolify.git
synced 2026-03-11 08:55:47 +00:00
5c50ab6162
Add runner execution observability and lifecycle hardening for self-hosted GitHub Actions runners, including: - scheduled artifact cleanup job for cached runner tarballs/templates - workflow_job in_progress handling to mark executions as running - safer cleanup failure handling and non-functional server fallback - persisted workflow job HTML URLs with execution "Open" links in UI - configurable runner group name sync (UI + provisioning + GitHub API) - webhook events persistence and auto-fix for missing required events - strict enum/status checks and related model cast/query improvements Also includes migrations and expanded feature/unit coverage for webhook, provisioning, cleanup, and runner group/event sync behaviors.
329 lines
12 KiB
PHP
329 lines
12 KiB
PHP
<?php
|
|
|
|
use App\Enums\GithubRunnerStatus;
|
|
use App\Jobs\ProvisionGithubRunnerJob;
|
|
use App\Models\GithubApp;
|
|
use App\Models\GithubRunnerConfig;
|
|
use App\Models\GithubRunnerExecution;
|
|
use App\Models\Server;
|
|
use App\Models\Team;
|
|
use Illuminate\Foundation\Testing\RefreshDatabase;
|
|
use Illuminate\Support\Facades\Http;
|
|
use Illuminate\Support\Facades\Queue;
|
|
|
|
uses(RefreshDatabase::class);
|
|
|
|
function makeRunnerSetup(array $configOverrides = []): array
|
|
{
|
|
$team = Team::factory()->create();
|
|
$privateKeyId = \Illuminate\Support\Facades\DB::table('private_keys')->insertGetId([
|
|
'uuid' => fake()->uuid(),
|
|
'name' => 'test-key',
|
|
'private_key' => encrypt('test'),
|
|
'team_id' => $team->id,
|
|
'created_at' => now(),
|
|
'updated_at' => now(),
|
|
]);
|
|
$server = Server::factory()->create(['private_key_id' => $privateKeyId, 'team_id' => $team->id]);
|
|
|
|
// Make server functional — Server::created() auto-creates settings with is_reachable=false.
|
|
// force_disabled must be explicitly set because it's not cast to boolean on ServerSetting.
|
|
$server->settings()->update(['is_reachable' => true, 'is_usable' => true, 'force_disabled' => false]);
|
|
$server->refresh();
|
|
|
|
$githubApp = GithubApp::create([
|
|
'name' => 'test-app',
|
|
'app_id' => fake()->unique()->randomNumber(6, true),
|
|
'installation_id' => 789,
|
|
'client_id' => 'Iv1.abc123',
|
|
'client_secret' => 'secret',
|
|
'webhook_secret' => 'test-secret',
|
|
'private_key_id' => $privateKeyId,
|
|
'team_id' => $team->id,
|
|
'api_url' => 'https://api.github.com',
|
|
'html_url' => 'https://github.com',
|
|
'organization' => 'test-org',
|
|
]);
|
|
$config = GithubRunnerConfig::create(array_merge([
|
|
'server_id' => $server->id,
|
|
'github_app_id' => $githubApp->id,
|
|
'labels' => ['self-hosted', 'coolify'],
|
|
'max_runners' => 2,
|
|
'capacity_wait_timeout' => 60,
|
|
], $configOverrides));
|
|
|
|
return compact('team', 'server', 'githubApp', 'config');
|
|
}
|
|
|
|
function makeJob(GithubApp $githubApp, array $overrides = []): ProvisionGithubRunnerJob
|
|
{
|
|
return new ProvisionGithubRunnerJob(
|
|
githubAppId: $githubApp->id,
|
|
workflowJobPayload: array_merge([
|
|
'id' => fake()->unique()->randomNumber(8, true),
|
|
'labels' => ['self-hosted', 'coolify'],
|
|
'workflow_name' => 'CI',
|
|
], $overrides['payload'] ?? []),
|
|
organizationLogin: 'test-org',
|
|
repositoryId: 0,
|
|
repositoryFullName: $overrides['repositoryFullName'] ?? null,
|
|
capacityWaitStartedAt: $overrides['capacityWaitStartedAt'] ?? null,
|
|
);
|
|
}
|
|
|
|
it('does not create an execution when no config matches the requested labels', function () {
|
|
Queue::fake();
|
|
['githubApp' => $githubApp] = makeRunnerSetup(['labels' => ['self-hosted', 'coolify']]);
|
|
|
|
$job = makeJob($githubApp, ['payload' => ['id' => 99001, 'labels' => ['self-hosted', 'gpu'], 'workflow_name' => 'CI']]);
|
|
$job->handle();
|
|
|
|
expect(GithubRunnerExecution::where('workflow_job_id', 99001)->exists())->toBeFalse();
|
|
Queue::assertNotPushed(ProvisionGithubRunnerJob::class);
|
|
});
|
|
|
|
it('re-dispatches with a delay when all matching configs are at capacity', function () {
|
|
Queue::fake();
|
|
['githubApp' => $githubApp, 'config' => $config, 'server' => $server] = makeRunnerSetup(['max_runners' => 1]);
|
|
|
|
// Fill the single runner slot
|
|
GithubRunnerExecution::create([
|
|
'server_id' => $server->id,
|
|
'github_runner_config_id' => $config->id,
|
|
'status' => GithubRunnerStatus::Running,
|
|
'runner_name' => 'coolify-existing',
|
|
'runner_dir' => '/opt/github-runners/coolify-existing',
|
|
'workflow_job_id' => 88001,
|
|
'pid' => 12345,
|
|
'started_at' => now()->subMinutes(5),
|
|
]);
|
|
|
|
$job = makeJob($githubApp, ['payload' => ['id' => 88002, 'labels' => ['self-hosted', 'coolify'], 'workflow_name' => 'CI']]);
|
|
$job->handle();
|
|
|
|
expect(GithubRunnerExecution::where('workflow_job_id', 88002)->exists())->toBeFalse();
|
|
|
|
Queue::assertPushed(ProvisionGithubRunnerJob::class, function ($pushedJob) {
|
|
return $pushedJob->capacityWaitStartedAt !== null
|
|
&& $pushedJob->workflowJobPayload['id'] === 88002;
|
|
});
|
|
});
|
|
|
|
it('preserves the original capacityWaitStartedAt when re-dispatching', function () {
|
|
Queue::fake();
|
|
['githubApp' => $githubApp, 'config' => $config, 'server' => $server] = makeRunnerSetup(['max_runners' => 1]);
|
|
|
|
GithubRunnerExecution::create([
|
|
'server_id' => $server->id,
|
|
'github_runner_config_id' => $config->id,
|
|
'status' => GithubRunnerStatus::Running,
|
|
'runner_name' => 'coolify-existing',
|
|
'runner_dir' => '/opt/github-runners/coolify-existing',
|
|
'workflow_job_id' => 77001,
|
|
'pid' => 12345,
|
|
'started_at' => now()->subMinutes(5),
|
|
]);
|
|
|
|
$originalStart = now()->subMinutes(30)->toIso8601String();
|
|
$job = makeJob($githubApp, [
|
|
'payload' => ['id' => 77002, 'labels' => ['self-hosted', 'coolify'], 'workflow_name' => 'CI'],
|
|
'capacityWaitStartedAt' => $originalStart,
|
|
]);
|
|
$job->handle();
|
|
|
|
Queue::assertPushed(ProvisionGithubRunnerJob::class, function ($pushedJob) use ($originalStart) {
|
|
return $pushedJob->capacityWaitStartedAt === $originalStart;
|
|
});
|
|
});
|
|
|
|
it('gives up silently when the capacity wait timeout is exceeded', function () {
|
|
Queue::fake();
|
|
|
|
['githubApp' => $githubApp, 'config' => $config, 'server' => $server] = makeRunnerSetup([
|
|
'max_runners' => 1,
|
|
'capacity_wait_timeout' => 60,
|
|
]);
|
|
|
|
GithubRunnerExecution::create([
|
|
'server_id' => $server->id,
|
|
'github_runner_config_id' => $config->id,
|
|
'status' => GithubRunnerStatus::Running,
|
|
'runner_name' => 'coolify-existing',
|
|
'runner_dir' => '/opt/github-runners/coolify-existing',
|
|
'workflow_job_id' => 66001,
|
|
'pid' => 12345,
|
|
'started_at' => now()->subHours(2),
|
|
]);
|
|
|
|
$job = makeJob($githubApp, [
|
|
'payload' => ['id' => 66002, 'labels' => ['self-hosted', 'coolify'], 'workflow_name' => 'CI'],
|
|
'capacityWaitStartedAt' => now()->subMinutes(61)->toIso8601String(),
|
|
]);
|
|
$job->handle();
|
|
|
|
expect(GithubRunnerExecution::where('workflow_job_id', 66002)->exists())->toBeFalse();
|
|
Queue::assertNotPushed(ProvisionGithubRunnerJob::class);
|
|
});
|
|
|
|
it('uses the configured timeout value for the capacity wait', function () {
|
|
Queue::fake();
|
|
|
|
['githubApp' => $githubApp, 'config' => $config, 'server' => $server] = makeRunnerSetup([
|
|
'max_runners' => 1,
|
|
'capacity_wait_timeout' => 10, // 10-minute custom timeout
|
|
]);
|
|
|
|
GithubRunnerExecution::create([
|
|
'server_id' => $server->id,
|
|
'github_runner_config_id' => $config->id,
|
|
'status' => GithubRunnerStatus::Running,
|
|
'runner_name' => 'coolify-existing',
|
|
'runner_dir' => '/opt/github-runners/coolify-existing',
|
|
'workflow_job_id' => 55001,
|
|
'pid' => 12345,
|
|
'started_at' => now()->subMinutes(15),
|
|
]);
|
|
|
|
// Started 11 minutes ago — exceeds 10-minute custom timeout
|
|
$job = makeJob($githubApp, [
|
|
'payload' => ['id' => 55002, 'labels' => ['self-hosted', 'coolify'], 'workflow_name' => 'CI'],
|
|
'capacityWaitStartedAt' => now()->subMinutes(11)->toIso8601String(),
|
|
]);
|
|
$job->handle();
|
|
|
|
expect(GithubRunnerExecution::where('workflow_job_id', 55002)->exists())->toBeFalse();
|
|
Queue::assertNotPushed(ProvisionGithubRunnerJob::class);
|
|
});
|
|
|
|
it('still re-dispatches when wait time is within the custom timeout', function () {
|
|
Queue::fake();
|
|
|
|
['githubApp' => $githubApp, 'config' => $config, 'server' => $server] = makeRunnerSetup([
|
|
'max_runners' => 1,
|
|
'capacity_wait_timeout' => 10,
|
|
]);
|
|
|
|
GithubRunnerExecution::create([
|
|
'server_id' => $server->id,
|
|
'github_runner_config_id' => $config->id,
|
|
'status' => GithubRunnerStatus::Running,
|
|
'runner_name' => 'coolify-existing',
|
|
'runner_dir' => '/opt/github-runners/coolify-existing',
|
|
'workflow_job_id' => 44001,
|
|
'pid' => 12345,
|
|
'started_at' => now()->subMinutes(5),
|
|
]);
|
|
|
|
// Started 5 minutes ago — within 10-minute timeout
|
|
$job = makeJob($githubApp, [
|
|
'payload' => ['id' => 44002, 'labels' => ['self-hosted', 'coolify'], 'workflow_name' => 'CI'],
|
|
'capacityWaitStartedAt' => now()->subMinutes(5)->toIso8601String(),
|
|
]);
|
|
$job->handle();
|
|
|
|
Queue::assertPushed(ProvisionGithubRunnerJob::class, fn ($j) => $j->workflowJobPayload['id'] === 44002);
|
|
});
|
|
|
|
it('counts cleaning executions as active capacity', function () {
|
|
['config' => $config, 'server' => $server] = makeRunnerSetup(['max_runners' => 1]);
|
|
|
|
GithubRunnerExecution::create([
|
|
'server_id' => $server->id,
|
|
'github_runner_config_id' => $config->id,
|
|
'status' => GithubRunnerStatus::Cleaning,
|
|
'runner_name' => 'coolify-cleaning',
|
|
'runner_dir' => '/opt/github-runners/coolify-cleaning',
|
|
'workflow_job_id' => 41001,
|
|
'pid' => 12345,
|
|
'started_at' => now()->subMinutes(1),
|
|
]);
|
|
|
|
expect($config->fresh()->activeRunnerCount())->toBe(1)
|
|
->and($config->fresh()->hasCapacity())->toBeFalse();
|
|
});
|
|
|
|
it('does not re-dispatch when the job has already been provisioned (idempotency)', function () {
|
|
Queue::fake();
|
|
['githubApp' => $githubApp, 'config' => $config, 'server' => $server] = makeRunnerSetup();
|
|
|
|
GithubRunnerExecution::create([
|
|
'server_id' => $server->id,
|
|
'github_runner_config_id' => $config->id,
|
|
'status' => GithubRunnerStatus::Running,
|
|
'runner_name' => 'coolify-existing',
|
|
'runner_dir' => '/opt/github-runners/coolify-existing',
|
|
'workflow_job_id' => 33001,
|
|
'pid' => 99,
|
|
'started_at' => now(),
|
|
]);
|
|
|
|
$job = makeJob($githubApp, ['payload' => ['id' => 33001, 'labels' => ['self-hosted', 'coolify'], 'workflow_name' => 'CI']]);
|
|
$job->handle();
|
|
|
|
// Should exit early — no new jobs dispatched, existing execution count unchanged
|
|
expect(GithubRunnerExecution::where('workflow_job_id', 33001)->count())->toBe(1);
|
|
Queue::assertNotPushed(ProvisionGithubRunnerJob::class);
|
|
});
|
|
|
|
it('does not re-dispatch when github reports the workflow job as cancelled', function () {
|
|
Queue::fake();
|
|
Http::fake([
|
|
'https://api.github.com/repos/test-org/test-repo/actions/jobs/21002' => Http::response([
|
|
'status' => 'completed',
|
|
'conclusion' => 'cancelled',
|
|
], 200),
|
|
]);
|
|
|
|
['githubApp' => $githubApp, 'config' => $config, 'server' => $server] = makeRunnerSetup(['max_runners' => 1]);
|
|
$githubApp->update(['is_public' => true]);
|
|
|
|
GithubRunnerExecution::create([
|
|
'server_id' => $server->id,
|
|
'github_runner_config_id' => $config->id,
|
|
'status' => GithubRunnerStatus::Running,
|
|
'runner_name' => 'coolify-existing',
|
|
'runner_dir' => '/opt/github-runners/coolify-existing',
|
|
'workflow_job_id' => 21001,
|
|
'pid' => 12345,
|
|
'started_at' => now()->subMinutes(2),
|
|
]);
|
|
|
|
$job = makeJob($githubApp, [
|
|
'payload' => ['id' => 21002, 'labels' => ['self-hosted', 'coolify'], 'workflow_name' => 'CI'],
|
|
'repositoryFullName' => 'test-org/test-repo',
|
|
]);
|
|
$job->handle();
|
|
|
|
expect(GithubRunnerExecution::where('workflow_job_id', 21002)->exists())->toBeFalse();
|
|
Queue::assertNotPushed(ProvisionGithubRunnerJob::class);
|
|
});
|
|
|
|
it('does not re-dispatch when github reports the workflow job as missing', function () {
|
|
Queue::fake();
|
|
Http::fake([
|
|
'https://api.github.com/repos/test-org/test-repo/actions/jobs/20002' => Http::response([], 404),
|
|
]);
|
|
|
|
['githubApp' => $githubApp, 'config' => $config, 'server' => $server] = makeRunnerSetup(['max_runners' => 1]);
|
|
$githubApp->update(['is_public' => true]);
|
|
|
|
GithubRunnerExecution::create([
|
|
'server_id' => $server->id,
|
|
'github_runner_config_id' => $config->id,
|
|
'status' => GithubRunnerStatus::Running,
|
|
'runner_name' => 'coolify-existing',
|
|
'runner_dir' => '/opt/github-runners/coolify-existing',
|
|
'workflow_job_id' => 20001,
|
|
'pid' => 12345,
|
|
'started_at' => now()->subMinutes(2),
|
|
]);
|
|
|
|
$job = makeJob($githubApp, [
|
|
'payload' => ['id' => 20002, 'labels' => ['self-hosted', 'coolify'], 'workflow_name' => 'CI'],
|
|
'repositoryFullName' => 'test-org/test-repo',
|
|
]);
|
|
$job->handle();
|
|
|
|
expect(GithubRunnerExecution::where('workflow_job_id', 20002)->exists())->toBeFalse();
|
|
Queue::assertNotPushed(ProvisionGithubRunnerJob::class);
|
|
});
|