From 2664a1a1719c364a4aea90702640c19d38306803 Mon Sep 17 00:00:00 2001 From: David Anyatonwu Date: Thu, 12 Dec 2024 15:54:31 +0100 Subject: [PATCH] fix: queue deployment on multiple app from same repo sometimes failed Signed-off-by: David Anyatonwu --- app/Jobs/ApplicationDeploymentJob.php | 351 ++++++++++++++++++-------- bootstrap/helpers/applications.php | 211 ++++++++++------ 2 files changed, 384 insertions(+), 178 deletions(-) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 6b677fa0e..16d4b32a4 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -26,6 +26,7 @@ use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Collection; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Process; use Illuminate\Support\Sleep; use Illuminate\Support\Str; @@ -170,12 +171,14 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue { $this->onQueue('high'); - $this->application_deployment_queue = ApplicationDeploymentQueue::find($application_deployment_queue_id); - $this->application = Application::find($this->application_deployment_queue->application_id); - $this->build_pack = data_get($this->application, 'build_pack'); - $this->build_args = collect([]); - $this->application_deployment_queue_id = $application_deployment_queue_id; + + // Load the deployment queue record + $this->application_deployment_queue = ApplicationDeploymentQueue::find($application_deployment_queue_id); + if (! $this->application_deployment_queue) { + throw new Exception('Deployment queue not found.'); + } + $this->deployment_uuid = $this->application_deployment_queue->deployment_uuid; $this->pull_request_id = $this->application_deployment_queue->pull_request_id; $this->commit = $this->application_deployment_queue->commit; @@ -186,16 +189,255 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue $this->force_rebuild = true; } $this->restart_only = $this->application_deployment_queue->restart_only; - $this->restart_only = $this->restart_only && $this->application->build_pack !== 'dockerimage' && $this->application->build_pack !== 'dockerfile'; $this->only_this_server = $this->application_deployment_queue->only_this_server; + // Load the application + $this->application = Application::find($this->application_deployment_queue->application_id); + if (! $this->application) { + throw new Exception('Application not found.'); + } + + $this->restart_only = $this->restart_only && $this->application->build_pack !== 'dockerimage' && $this->application->build_pack !== 'dockerfile'; + $this->git_type = data_get($this->application_deployment_queue, 'git_type'); $source = data_get($this->application, 'source'); if ($source) { $this->source = $source->getMorphClass()::where('id', $this->application->source->id)->first(); } + + // Load the server $this->server = Server::find($this->application_deployment_queue->server_id); + if (! $this->server) { + throw new Exception('Server not found.'); + } + + $this->timeout = $this->server->settings->dynamic_timeout; + $this->destination = $this->server->destinations()->where('id', $this->application_deployment_queue->destination_id)->first(); + if (! $this->destination) { + throw new Exception('Destination not found.'); + } + + $this->server = $this->mainServer = $this->destination->server; + $this->serverUser = $this->server->user; + $this->is_this_additional_server = $this->application->additional_servers()->wherePivot('server_id', $this->server->id)->count() > 0; + $this->preserveRepository = $this->application->settings->is_preserve_repository_enabled; + + $this->basedir = $this->application->generateBaseDir($this->deployment_uuid); + $this->workdir = "{$this->basedir}".rtrim($this->application->base_directory, '/'); + $this->configuration_dir = application_configuration_dir()."/{$this->application->uuid}"; + $this->is_debug_enabled = $this->application->settings->is_debug_enabled; + + $this->container_name = generateApplicationContainerName($this->application, $this->pull_request_id); + if ($this->application->settings->custom_internal_name && ! $this->application->settings->is_consistent_container_name_enabled) { + if ($this->pull_request_id === 0) { + $this->container_name = $this->application->settings->custom_internal_name; + } else { + $this->container_name = "{$this->application->settings->custom_internal_name}-pr-{$this->pull_request_id}"; + } + } + + $this->saved_outputs = collect(); + + // Set preview fqdn + if ($this->pull_request_id !== 0) { + $this->preview = $this->application->generate_preview_fqdn($this->pull_request_id); + if ($this->application->is_github_based()) { + ApplicationPullRequestUpdateJob::dispatch( + application: $this->application, + preview: $this->preview, + deployment_uuid: $this->deployment_uuid, + status: ProcessStatus::IN_PROGRESS + ); + } + if ($this->application->build_pack === 'dockerfile') { + if (data_get($this->application, 'dockerfile_location')) { + $this->dockerfile_location = $this->application->dockerfile_location; + } + } + } + } + + public function tags(): array + { + return ['server:'.gethostname()]; + } + + public function handle(): void + { + $cleanup_attempted = false; + + try { + DB::beginTransaction(); + + $this->application_deployment_queue = ApplicationDeploymentQueue::lockForUpdate()->find($this->application_deployment_queue_id); + if (! $this->application_deployment_queue) { + throw new Exception('Deployment queue not found.'); + } + + // Verify deployment is still valid + if ($this->application_deployment_queue->status !== ApplicationDeploymentStatus::IN_PROGRESS->value) { + throw new Exception('Deployment is no longer in progress.'); + } + + $this->application = Application::find($this->application_deployment_queue->application_id); + if (! $this->application) { + throw new Exception('Application not found.'); + } + + $this->server = $this->application_deployment_queue->server; + if (! $this->server) { + throw new Exception('Server not found.'); + } + + if ($this->server->isFunctional() === false) { + $this->application_deployment_queue->addLogEntry('Server is not functional.'); + throw new Exception('Server is not functional.'); + } + + try { + // Generate custom host<->ip mapping + $allContainers = instant_remote_process(["docker network inspect {$this->destination->network} -f '{{json .Containers}}' "], $this->server); + + if (! is_null($allContainers)) { + $allContainers = format_docker_command_output_to_json($allContainers); + $ips = collect([]); + if (count($allContainers) > 0) { + $allContainers = $allContainers[0]; + $allContainers = collect($allContainers)->sort()->values(); + foreach ($allContainers as $container) { + $containerName = data_get($container, 'Name'); + if ($containerName === 'coolify-proxy') { + continue; + } + if (preg_match('/-(\d{12})/', $containerName)) { + continue; + } + $containerIp = data_get($container, 'IPv4Address'); + if ($containerName && $containerIp) { + $containerIp = str($containerIp)->before('/'); + $ips->put($containerName, $containerIp->value()); + } + } + } + $this->addHosts = $ips->map(function ($ip, $name) { + return "--add-host $name:$ip"; + })->implode(' '); + } + + if ($this->application->dockerfile_target_build) { + $this->buildTarget = " --target {$this->application->dockerfile_target_build} "; + } + + // Check custom port + ['repository' => $this->customRepository, 'port' => $this->customPort] = $this->application->customRepository(); + + if (data_get($this->application, 'settings.is_build_server_enabled')) { + $teamId = data_get($this->application, 'environment.project.team.id'); + $buildServers = Server::buildServers($teamId)->get(); + if ($buildServers->count() === 0) { + $this->application_deployment_queue->addLogEntry('No suitable build server found. Using the deployment server.'); + $this->build_server = $this->server; + $this->original_server = $this->server; + } else { + $this->build_server = $buildServers->random(); + $this->application_deployment_queue->build_server_id = $this->build_server->id; + $this->application_deployment_queue->addLogEntry("Found a suitable build server ({$this->build_server->name})."); + $this->original_server = $this->server; + $this->use_build_server = true; + } + } else { + // Set build server & original_server to the same as deployment server + $this->build_server = $this->server; + $this->original_server = $this->server; + } + } catch (Exception $e) { + throw new Exception('Failed to initialize deployment: '.$e->getMessage()); + } + + DB::commit(); + + // Ensure cleanup happens in finally block + $this->decide_what_to_do(); + + // Mark deployment as successful + $this->application_deployment_queue->update([ + 'status' => ApplicationDeploymentStatus::FINISHED->value, + ]); + + ApplicationStatusChanged::dispatch(data_get($this->application, 'environment.project.team.id')); + + } catch (Exception $e) { + DB::rollBack(); + + if ($this->pull_request_id !== 0 && $this->application->is_github_based()) { + ApplicationPullRequestUpdateJob::dispatch( + application: $this->application, + preview: $this->preview, + deployment_uuid: $this->deployment_uuid, + status: ProcessStatus::ERROR + ); + } + + // Mark deployment as failed + if (isset($this->application_deployment_queue)) { + $this->application_deployment_queue->update([ + 'status' => ApplicationDeploymentStatus::FAILED->value, + 'error_message' => $e->getMessage(), + ]); + } + + $this->fail($e); + throw $e; + } finally { + try { + $cleanup_attempted = true; + + // Only cleanup if deployment was actually started + if ($this->deployment_uuid) { + if ($this->use_build_server) { + $this->server = $this->build_server; + } else { + $this->write_deployment_configurations(); + } + + // Check if container exists before removing + $this->execute_remote_command([ + "docker ps -q -f name={$this->deployment_uuid} && docker rm -f {$this->deployment_uuid} >/dev/null 2>&1", + 'hidden' => true, + 'ignore_errors' => true, + ]); + + // Clean up any temporary files + $this->execute_remote_command([ + "rm -rf {$this->basedir}", + 'hidden' => true, + 'ignore_errors' => true, + ]); + } + } catch (Exception $e) { + // Log cleanup failure but don't fail the deployment + \Log::error("Cleanup failed for deployment {$this->deployment_uuid}: ".$e->getMessage()); + } finally { + if (! $cleanup_attempted) { + // Ensure next deployment is queued even if cleanup fails + queue_next_deployment($this->application); + } + } + } + } + + private function initialize_deployment() + { + $this->build_pack = data_get($this->application, 'build_pack'); + $this->build_args = collect([]); + + $this->git_type = data_get($this->application_deployment_queue, 'git_type'); + + $source = data_get($this->application, 'source'); + if ($source) { + $this->source = $source->getMorphClass()::where('id', $this->application->source->id)->first(); + } $this->timeout = $this->server->settings->dynamic_timeout; $this->destination = $this->server->destinations()->where('id', $this->application_deployment_queue->destination_id)->first(); $this->server = $this->mainServer = $this->destination->server; @@ -233,103 +475,6 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue } } - public function tags(): array - { - return ['server:'.gethostname()]; - } - - public function handle(): void - { - $this->application_deployment_queue->update([ - 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, - ]); - if ($this->server->isFunctional() === false) { - $this->application_deployment_queue->addLogEntry('Server is not functional.'); - $this->fail('Server is not functional.'); - - return; - } - try { - // Generate custom host<->ip mapping - $allContainers = instant_remote_process(["docker network inspect {$this->destination->network} -f '{{json .Containers}}' "], $this->server); - - if (! is_null($allContainers)) { - $allContainers = format_docker_command_output_to_json($allContainers); - $ips = collect([]); - if (count($allContainers) > 0) { - $allContainers = $allContainers[0]; - $allContainers = collect($allContainers)->sort()->values(); - foreach ($allContainers as $container) { - $containerName = data_get($container, 'Name'); - if ($containerName === 'coolify-proxy') { - continue; - } - if (preg_match('/-(\d{12})/', $containerName)) { - continue; - } - $containerIp = data_get($container, 'IPv4Address'); - if ($containerName && $containerIp) { - $containerIp = str($containerIp)->before('/'); - $ips->put($containerName, $containerIp->value()); - } - } - } - $this->addHosts = $ips->map(function ($ip, $name) { - return "--add-host $name:$ip"; - })->implode(' '); - } - - if ($this->application->dockerfile_target_build) { - $this->buildTarget = " --target {$this->application->dockerfile_target_build} "; - } - - // Check custom port - ['repository' => $this->customRepository, 'port' => $this->customPort] = $this->application->customRepository(); - - if (data_get($this->application, 'settings.is_build_server_enabled')) { - $teamId = data_get($this->application, 'environment.project.team.id'); - $buildServers = Server::buildServers($teamId)->get(); - if ($buildServers->count() === 0) { - $this->application_deployment_queue->addLogEntry('No suitable build server found. Using the deployment server.'); - $this->build_server = $this->server; - $this->original_server = $this->server; - } else { - $this->build_server = $buildServers->random(); - $this->application_deployment_queue->build_server_id = $this->build_server->id; - $this->application_deployment_queue->addLogEntry("Found a suitable build server ({$this->build_server->name})."); - $this->original_server = $this->server; - $this->use_build_server = true; - } - } else { - // Set build server & original_server to the same as deployment server - $this->build_server = $this->server; - $this->original_server = $this->server; - } - $this->decide_what_to_do(); - } catch (Exception $e) { - if ($this->pull_request_id !== 0 && $this->application->is_github_based()) { - ApplicationPullRequestUpdateJob::dispatch(application: $this->application, preview: $this->preview, deployment_uuid: $this->deployment_uuid, status: ProcessStatus::ERROR); - } - $this->fail($e); - throw $e; - } finally { - if ($this->use_build_server) { - $this->server = $this->build_server; - } else { - $this->write_deployment_configurations(); - } - $this->execute_remote_command( - [ - "docker rm -f {$this->deployment_uuid} >/dev/null 2>&1", - 'hidden' => true, - 'ignore_errors' => true, - ] - ); - - ApplicationStatusChanged::dispatch(data_get($this->application, 'environment.project.team.id')); - } - } - private function decide_what_to_do() { if ($this->restart_only) { diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php index 73d5389ae..3f206f702 100644 --- a/bootstrap/helpers/applications.php +++ b/bootstrap/helpers/applications.php @@ -6,119 +6,180 @@ use App\Models\Application; use App\Models\ApplicationDeploymentQueue; use App\Models\Server; use App\Models\StandaloneDocker; +use Illuminate\Support\Facades\DB; use Spatie\Url\Url; function queue_application_deployment(Application $application, string $deployment_uuid, ?int $pull_request_id = 0, string $commit = 'HEAD', bool $force_rebuild = false, bool $is_webhook = false, bool $is_api = false, bool $restart_only = false, ?string $git_type = null, bool $no_questions_asked = false, ?Server $server = null, ?StandaloneDocker $destination = null, bool $only_this_server = false, bool $rollback = false) { - $application_id = $application->id; - $deployment_link = Url::fromString($application->link()."/deployment/{$deployment_uuid}"); - $deployment_url = $deployment_link->getPath(); - $server_id = $application->destination->server->id; - $server_name = $application->destination->server->name; - $destination_id = $application->destination->id; + return DB::transaction(function () use ($application, $deployment_uuid, $pull_request_id, $commit, $force_rebuild, $is_webhook, $is_api, $restart_only, $git_type, $no_questions_asked, $server, $destination, $only_this_server, $rollback) { + $application_id = $application->id; + $deployment_link = Url::fromString($application->link()."/deployment/{$deployment_uuid}"); + $deployment_url = $deployment_link->getPath(); - if ($server) { - $server_id = $server->id; - $server_name = $server->name; - } - if ($destination) { - $destination_id = $destination->id; - } - $deployment = ApplicationDeploymentQueue::create([ - 'application_id' => $application_id, - 'application_name' => $application->name, - 'server_id' => $server_id, - 'server_name' => $server_name, - 'destination_id' => $destination_id, - 'deployment_uuid' => $deployment_uuid, - 'deployment_url' => $deployment_url, - 'pull_request_id' => $pull_request_id, - 'force_rebuild' => $force_rebuild, - 'is_webhook' => $is_webhook, - 'is_api' => $is_api, - 'restart_only' => $restart_only, - 'commit' => $commit, - 'rollback' => $rollback, - 'git_type' => $git_type, - 'only_this_server' => $only_this_server, - ]); + $server_id = $application->destination->server->id; + $server_name = $application->destination->server->name; + $destination_id = $application->destination->id; - if ($no_questions_asked) { - ApplicationDeploymentJob::dispatch( - application_deployment_queue_id: $deployment->id, - ); - } elseif (next_queuable($server_id, $application_id)) { - ApplicationDeploymentJob::dispatch( - application_deployment_queue_id: $deployment->id, - ); - } + if ($server) { + $server_id = $server->id; + $server_name = $server->name; + } + if ($destination) { + $destination_id = $destination->id; + } + + // Check if there's already a deployment in progress for this application + $existing_deployment = ApplicationDeploymentQueue::where('application_id', $application_id) + ->whereIn('status', [ApplicationDeploymentStatus::IN_PROGRESS, ApplicationDeploymentStatus::QUEUED]) + ->lockForUpdate() + ->first(); + + if ($existing_deployment && ! $force_rebuild) { + throw new \RuntimeException('A deployment is already in progress or queued for this application.'); + } + + $deployment = ApplicationDeploymentQueue::create([ + 'application_id' => $application_id, + 'application_name' => $application->name, + 'server_id' => $server_id, + 'server_name' => $server_name, + 'destination_id' => $destination_id, + 'deployment_uuid' => $deployment_uuid, + 'deployment_url' => $deployment_url, + 'pull_request_id' => $pull_request_id, + 'force_rebuild' => $force_rebuild, + 'is_webhook' => $is_webhook, + 'is_api' => $is_api, + 'restart_only' => $restart_only, + 'commit' => $commit, + 'rollback' => $rollback, + 'git_type' => $git_type, + 'only_this_server' => $only_this_server, + 'status' => ApplicationDeploymentStatus::QUEUED, + ]); + + if ($no_questions_asked || next_queuable($server_id, $application_id)) { + $deployment->update([ + 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, + ]); + + ApplicationDeploymentJob::dispatch( + application_deployment_queue_id: $deployment->id + )->onQueue('high'); + } + + return $deployment; + }); } function force_start_deployment(ApplicationDeploymentQueue $deployment) { - $deployment->update([ - 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, - ]); + DB::transaction(function () use ($deployment) { + $deployment = ApplicationDeploymentQueue::lockForUpdate()->find($deployment->id); - ApplicationDeploymentJob::dispatch( - application_deployment_queue_id: $deployment->id, - ); -} -function queue_next_deployment(Application $application) -{ - $server_id = $application->destination->server_id; - $next_found = ApplicationDeploymentQueue::where('server_id', $server_id)->where('status', ApplicationDeploymentStatus::QUEUED)->get()->sortBy('created_at')->first(); - if ($next_found) { - $next_found->update([ + if (! $deployment) { + throw new \RuntimeException('Deployment not found.'); + } + + $deployment->update([ 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, ]); ApplicationDeploymentJob::dispatch( - application_deployment_queue_id: $next_found->id, - ); - } + application_deployment_queue_id: $deployment->id + )->onQueue('high'); + }); +} +function queue_next_deployment(Application $application) +{ + $server_id = $application->destination->server_id; + + // Use transaction to prevent race conditions + DB::transaction(function () use ($server_id) { + // Lock the queued deployments for update to prevent race conditions + $next_found = ApplicationDeploymentQueue::where('server_id', $server_id) + ->where('status', ApplicationDeploymentStatus::QUEUED) + ->orderBy('created_at') + ->lockForUpdate() + ->first(); + + if ($next_found) { + // Check if we can start this deployment + $server = Server::find($server_id); + $concurrent_builds = $server->settings->concurrent_builds; + + $in_progress_count = ApplicationDeploymentQueue::where('server_id', $server_id) + ->where('status', 'in_progress') + ->count(); + + if ($in_progress_count < $concurrent_builds) { + $next_found->update([ + 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, + ]); + + ApplicationDeploymentJob::dispatch( + application_deployment_queue_id: $next_found->id + )->onQueue('high'); + } + } + }); } function next_queuable(string $server_id, string $application_id): bool { - $deployments = ApplicationDeploymentQueue::where('server_id', $server_id)->whereIn('status', ['in_progress', ApplicationDeploymentStatus::QUEUED])->get()->sortByDesc('created_at'); + $deployments = ApplicationDeploymentQueue::where('server_id', $server_id) + ->whereIn('status', ['in_progress', ApplicationDeploymentStatus::QUEUED]) + ->get() + ->sortByDesc('created_at'); + + // Check if there are any deployments in progress for this application $same_application_deployments = $deployments->where('application_id', $application_id); - $in_progress = $same_application_deployments->filter(function ($value, $key) { + $in_progress = $same_application_deployments->filter(function ($value) { return $value->status === 'in_progress'; }); + if ($in_progress->count() > 0) { return false; } + $server = Server::find($server_id); $concurrent_builds = $server->settings->concurrent_builds; - // ray("serverId:{$server->id}", "concurrentBuilds:{$concurrent_builds}", "deployments:{$deployments->count()}", "sameApplicationDeployments:{$same_application_deployments->count()}")->green(); + // Count only in_progress deployments for concurrent limit + $in_progress_count = $deployments->filter(function ($value) { + return $value->status === 'in_progress'; + })->count(); - if ($deployments->count() > $concurrent_builds) { - return false; - } - - return true; + return $in_progress_count < $concurrent_builds; } function next_after_cancel(?Server $server = null) { if ($server) { - $next_found = ApplicationDeploymentQueue::where('server_id', data_get($server, 'id'))->where('status', ApplicationDeploymentStatus::QUEUED)->get()->sortBy('created_at'); - if ($next_found->count() > 0) { - foreach ($next_found as $next) { - $server = Server::find($next->server_id); + DB::transaction(function () use ($server) { + // Lock the queued deployments for update to prevent race conditions + $next_found = ApplicationDeploymentQueue::where('server_id', $server->id) + ->where('status', ApplicationDeploymentStatus::QUEUED) + ->orderBy('created_at') + ->lockForUpdate() + ->first(); + + if ($next_found) { + $server = Server::find($next_found->server_id); $concurrent_builds = $server->settings->concurrent_builds; - $inprogress_deployments = ApplicationDeploymentQueue::where('server_id', $next->server_id)->whereIn('status', [ApplicationDeploymentStatus::QUEUED])->get()->sortByDesc('created_at'); - if ($inprogress_deployments->count() < $concurrent_builds) { - $next->update([ + + $in_progress_count = ApplicationDeploymentQueue::where('server_id', $next_found->server_id) + ->where('status', ApplicationDeploymentStatus::IN_PROGRESS) + ->count(); + + if ($in_progress_count < $concurrent_builds) { + $next_found->update([ 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, ]); ApplicationDeploymentJob::dispatch( - application_deployment_queue_id: $next->id, - ); + application_deployment_queue_id: $next_found->id + )->onQueue('high'); } - break; } - } + }); } }