From 5a34f180ebcf07631d310a894a23731f1586e645 Mon Sep 17 00:00:00 2001 From: H01001000 Date: Fri, 12 Dec 2025 20:27:25 -0800 Subject: [PATCH 1/8] refactor: add caching for docker inspect results for ComplexStatusCheck --- app/Actions/Docker/GetContainersStatus.php | 3 ++- app/Actions/Shared/ComplexStatusCheck.php | 18 +++++++++++++++--- app/Jobs/PushServerUpdateJob.php | 3 ++- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/app/Actions/Docker/GetContainersStatus.php b/app/Actions/Docker/GetContainersStatus.php index a1476e120..f34c8e4c9 100644 --- a/app/Actions/Docker/GetContainersStatus.php +++ b/app/Actions/Docker/GetContainersStatus.php @@ -46,10 +46,11 @@ class GetContainersStatus } $this->applications = $this->server->applications(); $skip_these_applications = collect([]); + $dockerInspectCache = []; foreach ($this->applications as $application) { if ($application->additional_servers->count() > 0) { $skip_these_applications->push($application); - ComplexStatusCheck::run($application); + ComplexStatusCheck::run($application, $dockerInspectCache); $this->applications = $this->applications->filter(function ($value, $key) use ($application) { return $value->id !== $application->id; }); diff --git a/app/Actions/Shared/ComplexStatusCheck.php b/app/Actions/Shared/ComplexStatusCheck.php index 3649be986..3d37ea8d4 100644 --- a/app/Actions/Shared/ComplexStatusCheck.php +++ b/app/Actions/Shared/ComplexStatusCheck.php @@ -12,7 +12,7 @@ class ComplexStatusCheck use AsAction; use CalculatesExcludedStatus; - public function handle(Application $application) + public function handle(Application $application, array &$dockerInspectCache = []) { $servers = $application->additional_servers; $servers->push($application->destination->server); @@ -29,8 +29,20 @@ class ComplexStatusCheck continue; } } - $containers = instant_remote_process(["docker container inspect $(docker container ls -q --filter 'label=coolify.applicationId={$application->id}' --filter 'label=coolify.pullRequestId=0') --format '{{json .}}'"], $server, false); - $containers = format_docker_command_output_to_json($containers); + + if (!isset($dockerInspectCache[$server->id])) { + $allContainers = instant_remote_process(["docker container inspect $(docker ps -aq --filter 'label=coolify.pullRequestId=0') --format 'json'"], $server, false); + $allContainers = format_docker_command_output_to_json($allContainers); + $dockerInspectCache[$server->id] = $allContainers; + } + $allContainers = $dockerInspectCache[$server->id]; + + $containers = collect($allContainers)->filter(function ($container) use ($application) { + $labels = data_get($container, 'Config.Labels', []); + $appId = data_get($labels, 'coolify.applicationId'); + + return $appId && intval($appId) === $application->id; + }); if ($containers->count() > 0) { $statusToSet = $this->aggregateContainerStatuses($application, $containers); diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index e6c64ada7..29151e2d2 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -580,8 +580,9 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced private function updateAdditionalServersStatus() { + $dockerInspectCache = []; $this->allApplicationsWithAdditionalServers->each(function ($application) { - ComplexStatusCheck::run($application); + ComplexStatusCheck::run($application, $dockerInspectCache); }); } From 1305ba56b09038e5ff63b6f2b3c6b28584cc86f4 Mon Sep 17 00:00:00 2001 From: H01001000 Date: Fri, 12 Dec 2025 21:26:40 -0800 Subject: [PATCH 2/8] fix: use object instead, cause laravel dont pass array by ref --- app/Actions/Docker/GetContainersStatus.php | 3 ++- app/Actions/Shared/ComplexStatusCheck.php | 17 +++++++++++++---- app/Actions/Shared/DockerInspectCache.php | 8 ++++++++ app/Jobs/PushServerUpdateJob.php | 3 ++- 4 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 app/Actions/Shared/DockerInspectCache.php diff --git a/app/Actions/Docker/GetContainersStatus.php b/app/Actions/Docker/GetContainersStatus.php index f34c8e4c9..a144ece3c 100644 --- a/app/Actions/Docker/GetContainersStatus.php +++ b/app/Actions/Docker/GetContainersStatus.php @@ -4,6 +4,7 @@ namespace App\Actions\Docker; use App\Actions\Database\StartDatabaseProxy; use App\Actions\Shared\ComplexStatusCheck; +use App\Actions\Shared\DockerInspectCache; use App\Events\ServiceChecked; use App\Models\ApplicationPreview; use App\Models\Server; @@ -46,7 +47,7 @@ class GetContainersStatus } $this->applications = $this->server->applications(); $skip_these_applications = collect([]); - $dockerInspectCache = []; + $dockerInspectCache = new DockerInspectCache(); foreach ($this->applications as $application) { if ($application->additional_servers->count() > 0) { $skip_these_applications->push($application); diff --git a/app/Actions/Shared/ComplexStatusCheck.php b/app/Actions/Shared/ComplexStatusCheck.php index 3d37ea8d4..52905838e 100644 --- a/app/Actions/Shared/ComplexStatusCheck.php +++ b/app/Actions/Shared/ComplexStatusCheck.php @@ -5,15 +5,22 @@ namespace App\Actions\Shared; use App\Models\Application; use App\Services\ContainerStatusAggregator; use App\Traits\CalculatesExcludedStatus; +use App\Actions\Shared\DockerInspectCache; use Lorisleiva\Actions\Concerns\AsAction; +use Illuminate\Support\Facades\Log; class ComplexStatusCheck { use AsAction; use CalculatesExcludedStatus; - public function handle(Application $application, array &$dockerInspectCache = []) + public function handle(Application $application, DockerInspectCache $dockerInspectCache = new DockerInspectCache()) { + Log::info('Docker inspect cache size', [ + 'entries' => count($dockerInspectCache->data), + ]); + dump('Cache entries b4:', count($dockerInspectCache->data)); + $servers = $application->additional_servers; $servers->push($application->destination->server); foreach ($servers as $server) { @@ -30,12 +37,12 @@ class ComplexStatusCheck } } - if (!isset($dockerInspectCache[$server->id])) { + if (!isset($dockerInspectCache->data[$server->id])) { $allContainers = instant_remote_process(["docker container inspect $(docker ps -aq --filter 'label=coolify.pullRequestId=0') --format 'json'"], $server, false); $allContainers = format_docker_command_output_to_json($allContainers); - $dockerInspectCache[$server->id] = $allContainers; + $dockerInspectCache->data[$server->id] = $allContainers; } - $allContainers = $dockerInspectCache[$server->id]; + $allContainers = $dockerInspectCache->data[$server->id]; $containers = collect($allContainers)->filter(function ($container) use ($application) { $labels = data_get($container, 'Config.Labels', []); @@ -71,6 +78,8 @@ class ComplexStatusCheck } } } + + dump('Cache entries after:', count($dockerInspectCache->data)); } private function aggregateContainerStatuses($application, $containers) diff --git a/app/Actions/Shared/DockerInspectCache.php b/app/Actions/Shared/DockerInspectCache.php new file mode 100644 index 000000000..0f8303e56 --- /dev/null +++ b/app/Actions/Shared/DockerInspectCache.php @@ -0,0 +1,8 @@ +allApplicationsWithAdditionalServers->each(function ($application) { ComplexStatusCheck::run($application, $dockerInspectCache); }); From c268536a452391a2472e3df57bb09349ace81930 Mon Sep 17 00:00:00 2001 From: H01001000 Date: Sat, 13 Dec 2025 17:23:05 -0800 Subject: [PATCH 3/8] feat: make instant_remote_process work with multiple server same time --- bootstrap/helpers/remoteProcess.php | 69 ++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index bdfbaba48..117d19d2a 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -118,36 +118,63 @@ function instant_remote_process_with_timeout(Collection|array $command, Server $ ); } -function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false, ?int $timeout = null, bool $disableMultiplexing = false): ?string -{ +function instant_remote_process( + Collection|array $command, + Server|Collection|array $servers, + bool $throwError = true, + bool $no_sudo = false, + ?int $timeout = null, + bool $disableMultiplexing = false +): string|array|null { $command = $command instanceof Collection ? $command->toArray() : $command; - - if ($server->isNonRoot() && ! $no_sudo) { - $command = parseCommandsByLineForSudo(collect($command), $server); - } - $command_string = implode("\n", $command); - $effectiveTimeout = $timeout ?? config('constants.ssh.command_timeout'); + $servers = $servers instanceof Collection ? $servers : collect(is_array($servers) ? $servers : [$servers]); return \App\Helpers\SshRetryHandler::retry( - function () use ($server, $command_string, $effectiveTimeout, $disableMultiplexing) { - $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string, $disableMultiplexing); - $process = Process::timeout($effectiveTimeout)->run($sshCommand); + function () use ($servers, $command, $no_sudo, $timeout, $disableMultiplexing) { + $results = Process::concurrently(function ($pool) use ( + $servers, + $command, + $no_sudo, + $timeout, + $disableMultiplexing + ) { + foreach ($servers as $server) { + if ($server->isNonRoot() && ! $no_sudo) { + $command = parseCommandsByLineForSudo(collect($command), $server); + } + $command_string = implode("\n", $command); + $effectiveTimeout = $timeout ?? config('constants.ssh.command_timeout'); - $output = trim($process->output()); - $exitCode = $process->exitCode(); + $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string, $disableMultiplexing); - if ($exitCode !== 0) { - excludeCertainErrors($process->errorOutput(), $exitCode); + $pool->as($server->id)->timeout($effectiveTimeout)->command($sshCommand); + } + }); + + $outputs = []; + + foreach ($results->collect() as $serverId => $process) { + $output = trim($process->output()); + $exitCode = $process->exitCode(); + + if ($exitCode !== 0) { + excludeCertainErrors($process->errorOutput(), $exitCode); + } + + // Sanitize output to ensure valid UTF-8 encoding + $output = $output === 'null' ? null : sanitize_utf8_text($output); + $outputs[$serverId] = $output; } - // Sanitize output to ensure valid UTF-8 encoding - $output = $output === 'null' ? null : sanitize_utf8_text($output); - - return $output; + if ($servers->count() === 1) { + return $outputs[$servers->first()->id]; + } + + return $outputs; }, [ - 'server' => $server->ip, - 'command_preview' => substr($command_string, 0, 100), + 'server' => implode(', ', $servers->pluck('ip')->toArray()), + 'command_preview' => substr(implode("\n", $command), 0, 100), 'function' => 'instant_remote_process', ], $throwError From ebc9b193446e40dd7baa7f23242a1bd48a5f4fb5 Mon Sep 17 00:00:00 2001 From: H01001000 Date: Sat, 13 Dec 2025 17:23:44 -0800 Subject: [PATCH 4/8] chore: remove dev log --- app/Actions/Shared/ComplexStatusCheck.php | 7 ------- 1 file changed, 7 deletions(-) diff --git a/app/Actions/Shared/ComplexStatusCheck.php b/app/Actions/Shared/ComplexStatusCheck.php index 52905838e..70a985e6a 100644 --- a/app/Actions/Shared/ComplexStatusCheck.php +++ b/app/Actions/Shared/ComplexStatusCheck.php @@ -16,11 +16,6 @@ class ComplexStatusCheck public function handle(Application $application, DockerInspectCache $dockerInspectCache = new DockerInspectCache()) { - Log::info('Docker inspect cache size', [ - 'entries' => count($dockerInspectCache->data), - ]); - dump('Cache entries b4:', count($dockerInspectCache->data)); - $servers = $application->additional_servers; $servers->push($application->destination->server); foreach ($servers as $server) { @@ -78,8 +73,6 @@ class ComplexStatusCheck } } } - - dump('Cache entries after:', count($dockerInspectCache->data)); } private function aggregateContainerStatuses($application, $containers) From 3e5d18a202a7735402ba9f23e362899cf874ec16 Mon Sep 17 00:00:00 2001 From: H01001000 Date: Sat, 13 Dec 2025 17:24:10 -0800 Subject: [PATCH 5/8] refactor: change to run on all server same time --- app/Actions/Shared/ComplexStatusCheck.php | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/app/Actions/Shared/ComplexStatusCheck.php b/app/Actions/Shared/ComplexStatusCheck.php index 70a985e6a..1c8f84ac5 100644 --- a/app/Actions/Shared/ComplexStatusCheck.php +++ b/app/Actions/Shared/ComplexStatusCheck.php @@ -18,6 +18,17 @@ class ComplexStatusCheck { $servers = $application->additional_servers; $servers->push($application->destination->server); + + $serversToInspect = $servers->filter(fn($server) => !isset($dockerInspectCache->data[$server->id])); + + if ($serversToInspect->isNotEmpty()) { + $results = instant_remote_process(["docker container inspect $(docker container ls -aq) --format '{{json .}}'"], $serversToInspect, false); + + foreach ($results as $serverId => $result) { + $dockerInspectCache->data[$serverId] = format_docker_command_output_to_json($result); + } + } + foreach ($servers as $server) { $is_main_server = $application->destination->server->id === $server->id; if (! $server->isFunctional()) { @@ -31,19 +42,14 @@ class ComplexStatusCheck continue; } } - - if (!isset($dockerInspectCache->data[$server->id])) { - $allContainers = instant_remote_process(["docker container inspect $(docker ps -aq --filter 'label=coolify.pullRequestId=0') --format 'json'"], $server, false); - $allContainers = format_docker_command_output_to_json($allContainers); - $dockerInspectCache->data[$server->id] = $allContainers; - } $allContainers = $dockerInspectCache->data[$server->id]; $containers = collect($allContainers)->filter(function ($container) use ($application) { $labels = data_get($container, 'Config.Labels', []); - $appId = data_get($labels, 'coolify.applicationId'); + $appId = $labels['coolify.applicationId'] ?? null; + $pullRequestId = $labels['coolify.pullRequestId'] ?? null; - return $appId && intval($appId) === $application->id; + return $appId !== null && intval($appId) === $application->id && $pullRequestId !== null && intval($pullRequestId) === 0; }); if ($containers->count() > 0) { From ba5a497b06d64db23b3ed4480fb3d26e2894f347 Mon Sep 17 00:00:00 2001 From: H01001000 Date: Sat, 13 Dec 2025 17:24:35 -0800 Subject: [PATCH 6/8] chore: remove unused use --- app/Actions/Shared/ComplexStatusCheck.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/Actions/Shared/ComplexStatusCheck.php b/app/Actions/Shared/ComplexStatusCheck.php index 1c8f84ac5..118ae33be 100644 --- a/app/Actions/Shared/ComplexStatusCheck.php +++ b/app/Actions/Shared/ComplexStatusCheck.php @@ -7,7 +7,6 @@ use App\Services\ContainerStatusAggregator; use App\Traits\CalculatesExcludedStatus; use App\Actions\Shared\DockerInspectCache; use Lorisleiva\Actions\Concerns\AsAction; -use Illuminate\Support\Facades\Log; class ComplexStatusCheck { From f4c867a6a71418a15ea6736f2f2ded63edab5c39 Mon Sep 17 00:00:00 2001 From: H01001000 Date: Sat, 13 Dec 2025 17:24:50 -0800 Subject: [PATCH 7/8] refactor: use cached result --- app/Actions/Docker/GetContainersStatus.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/Actions/Docker/GetContainersStatus.php b/app/Actions/Docker/GetContainersStatus.php index a144ece3c..a64705a4f 100644 --- a/app/Actions/Docker/GetContainersStatus.php +++ b/app/Actions/Docker/GetContainersStatus.php @@ -61,7 +61,12 @@ class GetContainersStatus return ! $skip_these_applications->pluck('id')->contains($value->id); }); if ($this->containers === null) { - ['containers' => $this->containers, 'containerReplicates' => $this->containerReplicates] = $this->server->getContainers(); + if (isset($dockerInspectCache->data[$this->server->id]) && !$this->server->isSwarm()) { + $this->containers = collect($dockerInspectCache->data[$this->server->id]); + $this->containerReplicates = collect([]); + } else { + ['containers' => $this->containers, 'containerReplicates' => $this->containerReplicates] = $this->server->getContainers(); + } } if (is_null($this->containers)) { From 42c386f4f98ce1f38a08205640b2190b661b7d19 Mon Sep 17 00:00:00 2001 From: H01001000 Date: Sun, 14 Dec 2025 12:57:50 -0800 Subject: [PATCH 8/8] fix: update parameter name for backward compatibility in instant_remote_process --- bootstrap/helpers/remoteProcess.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 117d19d2a..0c1fecc64 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -120,14 +120,15 @@ function instant_remote_process_with_timeout(Collection|array $command, Server $ function instant_remote_process( Collection|array $command, - Server|Collection|array $servers, + Server|Collection|array $server, bool $throwError = true, bool $no_sudo = false, ?int $timeout = null, bool $disableMultiplexing = false ): string|array|null { $command = $command instanceof Collection ? $command->toArray() : $command; - $servers = $servers instanceof Collection ? $servers : collect(is_array($servers) ? $servers : [$servers]); + // Use the variable server instead of servers for backward compatibility + $servers = $server instanceof Collection ? $server : collect(is_array($server) ? $server : [$server]); return \App\Helpers\SshRetryHandler::retry( function () use ($servers, $command, $no_sudo, $timeout, $disableMultiplexing) {