Fix sporadic SSH "Permission denied (publickey)" errors

Add SSH key content validation to detect stale key files and
invalidate multiplexed connections when key content changes.

Fixes #7724
This commit is contained in:
KrE80r 2025-12-21 23:03:22 +10:30
parent d6864ce378
commit 3ff36ef684
2 changed files with 111 additions and 4 deletions

View file

@ -8,6 +8,7 @@ use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Hash;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Storage;
class SshMultiplexingHelper
{
@ -201,14 +202,52 @@ class SshMultiplexingHelper
return config('constants.ssh.mux_enabled') && ! config('constants.coolify.is_windows_docker_desktop');
}
/**
* Validate that the SSH key file exists and contains the correct content.
*
* This method addresses sporadic "Permission denied (publickey)" errors that can occur
* when the SSH key file content becomes stale or mismatched with the database.
*
* @see https://github.com/coollabsio/coolify/issues/7724
*/
private static function validateSshKey(PrivateKey $privateKey): void
{
$keyLocation = $privateKey->getKeyLocation();
$checkKeyCommand = "ls $keyLocation 2>/dev/null";
$keyCheckProcess = Process::run($checkKeyCommand);
$disk = Storage::disk('ssh-keys');
$filename = "ssh_key@{$privateKey->uuid}";
if ($keyCheckProcess->exitCode() !== 0) {
if (! $disk->exists($filename)) {
Log::debug('SSH key file not found, storing key', [
'key_uuid' => $privateKey->uuid,
]);
$privateKey->storeInFileSystem();
return;
}
// Verify file content matches database to prevent stale key issues
$storedContent = $disk->get($filename);
if ($storedContent !== $privateKey->private_key) {
Log::warning('SSH key file content mismatch detected, re-storing key', [
'key_uuid' => $privateKey->uuid,
]);
$privateKey->storeInFileSystem();
// Invalidate any multiplexed connections that may be using the old key
foreach ($privateKey->servers as $server) {
try {
self::removeMuxFile($server);
Log::debug('Invalidated mux connection due to key content mismatch', [
'server_uuid' => $server->uuid,
'key_uuid' => $privateKey->uuid,
]);
} catch (\Exception $e) {
Log::warning('Failed to invalidate mux connection during key validation', [
'server_uuid' => $server->uuid,
'key_uuid' => $privateKey->uuid,
'error' => $e->getMessage(),
]);
}
}
}
}

View file

@ -0,0 +1,68 @@
<?php
/**
* Unit tests for SSH key content validation fix.
*
* These tests verify the fix for issue #7724: Sporadic "Permission denied (publickey)" errors
* caused by SSH key content mismatch between database and filesystem.
*
* @see https://github.com/coollabsio/coolify/issues/7724
*/
use App\Helpers\SshMultiplexingHelper;
use App\Models\PrivateKey;
test('PrivateKey model encrypts private_key attribute', function () {
$privateKey = new PrivateKey;
$casts = $privateKey->getCasts();
expect($casts['private_key'])->toBe('encrypted');
});
test('PrivateKey getKeyLocation returns correct path format', function () {
$privateKey = new PrivateKey;
$privateKey->uuid = 'test-uuid-123';
expect($privateKey->getKeyLocation())->toBe('/var/www/html/storage/app/ssh/keys/ssh_key@test-uuid-123');
});
test('SshMultiplexingHelper validateSshKey method exists', function () {
$class = new ReflectionClass(SshMultiplexingHelper::class);
$method = $class->getMethod('validateSshKey');
expect($method->isPrivate())->toBeTrue()
->and($method->isStatic())->toBeTrue();
});
test('SshMultiplexingHelper uses Storage disk for key validation', function () {
$class = new ReflectionClass(SshMultiplexingHelper::class);
$source = file_get_contents($class->getFileName());
expect($source)->toContain("Storage::disk('ssh-keys')")
->and($source)->toContain('$disk->exists($filename)')
->and($source)->toContain('$disk->get($filename)');
});
test('SshMultiplexingHelper logs key content mismatch', function () {
$class = new ReflectionClass(SshMultiplexingHelper::class);
$source = file_get_contents($class->getFileName());
expect($source)->toContain("Log::warning('SSH key file content mismatch detected")
->and($source)->toContain("Log::debug('SSH key file not found, storing key");
});
test('validateSshKey compares file content with database value', function () {
$class = new ReflectionClass(SshMultiplexingHelper::class);
$source = file_get_contents($class->getFileName());
expect($source)->toContain('$storedContent !== $privateKey->private_key')
->and($source)->toContain('$privateKey->storeInFileSystem()');
});
test('validateSshKey invalidates mux on content mismatch', function () {
$class = new ReflectionClass(SshMultiplexingHelper::class);
$source = file_get_contents($class->getFileName());
expect($source)->toContain('self::removeMuxFile($server)')
->and($source)->toContain("Log::debug('Invalidated mux connection due to key content mismatch");
});