Add managed registry provisioning, pruning, and readiness tracking

2026-06-08 20:44:16 +01:00
parent 5b977c1f41
commit 3a851db08f
52 changed files with 2706 additions and 116 deletions
--- a/tests/Feature/ManagedRegistryTest.php
+++ b/tests/Feature/ManagedRegistryTest.php
@@ -0,0 +1,354 @@
+<?php
+
+use App\Actions\Registries\CreateManagedRegistryMaintenanceOperation;
+use App\Actions\Registries\CreateManagedRegistryProvisionOperation;
+use App\Actions\Registries\CreateManagedRegistrySmokeCheckOperation;
+use App\Actions\Registries\CreateRegistryAuthOperation;
+use App\Enums\BuildArtifactStatus;
+use App\Enums\OperationKind;
+use App\Enums\OperationStatus;
+use App\Jobs\Services\RunStep;
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\Network;
+use App\Models\Organisation;
+use App\Models\Provider;
+use App\Models\Server;
+use App\Models\Service;
+use App\Services\Operations\RemoteCommandRunner;
+use App\Services\Registries\ManagedRegistryHealth;
+use App\Services\Registries\ManagedRegistryProvisioner;
+use App\Services\Registries\ManagedRegistryRetention;
+use Illuminate\Support\Facades\Http;
+
+it('provisions a persisted managed registry with encrypted scoped credentials and defaults', function () {
+    config([
+        'keystone.managed_registry.storage_path' => '/mnt/registry',
+        'keystone.managed_registry.retention.successful_artifacts_per_environment' => 5,
+    ]);
+
+    $organisation = Organisation::factory()->create();
+    $server = managedRegistryServerFor($organisation);
+
+    $registry = app(ManagedRegistryProvisioner::class)->provision(
+        organisation: $organisation,
+        url: 'https://registry.example.com',
+        controlServer: $server,
+    );
+
+    expect($registry->url)->toBe('registry.example.com')
+        ->and($registry->storage_path)->toBe('/mnt/registry')
+        ->and($registry->retention_successful_artifacts)->toBe(5)
+        ->and($registry->control_server_id)->toBe($server->id)
+        ->and($registry->credentials)->toHaveKeys([
+            'build_username',
+            'build_password',
+            'runtime_username',
+            'runtime_password',
+        ])
+        ->and($registry->getRawOriginal('credentials'))->not->toContain($registry->credentials['build_password'])
+        ->and($server->refresh()->is_control_node)->toBeTrue()
+        ->and($server->build_enabled)->toBeTrue();
+});
+
+it('checks managed registry readiness over https and records health', function () {
+    Http::fake([
+        'https://registry.example.com/v2/' => Http::response('', 401),
+    ]);
+
+    $organisation = Organisation::factory()->create();
+    $server = managedRegistryServerFor($organisation);
+    $registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server);
+    $registry->forceFill([
+        'readiness_checks' => [
+            'control_https' => 'pending',
+            'build_push' => 'passed',
+            'runtime_pull_server_'.$server->id => 'passed',
+        ],
+    ])->save();
+
+    expect(app(ManagedRegistryHealth::class)->check($registry))->toBeTrue()
+        ->and($registry->refresh()->health_status)->toBe('healthy')
+        ->and($registry->ready_at)->not->toBeNull();
+});
+
+it('creates registry auth operations with encrypted secrets and placeholder scripts', function () {
+    $organisation = Organisation::factory()->create();
+    $server = managedRegistryServerFor($organisation);
+    $registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server);
+    $registry->markHealthy();
+
+    $operation = app(CreateRegistryAuthOperation::class)->execute($registry, $server, 'runtime');
+    $step = $operation->steps()->firstOrFail();
+
+    expect($operation->kind)->toBe(OperationKind::CREDENTIAL_ROTATION)
+        ->and($step->script)->toContain('docker login')
+        ->and($step->script)->toContain('[!registry_password_base64!]')
+        ->and($step->script)->not->toContain($registry->credentials['runtime_password'])
+        ->and($step->scriptForExecution())->toContain(base64_encode($registry->credentials['runtime_password']))
+        ->and($step->scriptForExecution())->not->toContain($registry->credentials['runtime_password'])
+        ->and($step->getRawOriginal('secrets'))->not->toContain($registry->credentials['runtime_password']);
+});
+
+it('creates a managed registry provision operation with registry service, htpasswd auth, storage, deletion, and caddy proxy without raw passwords', function () {
+    $organisation = Organisation::factory()->create();
+    $server = managedRegistryServerFor($organisation);
+    $registry = app(ManagedRegistryProvisioner::class)->provision(
+        organisation: $organisation,
+        url: 'https://registry.example.com',
+        controlServer: $server,
+        storagePath: '/mnt/keystone-registry',
+    );
+
+    $operation = app(CreateManagedRegistryProvisionOperation::class)->execute($registry);
+    $step = $operation->steps()->firstOrFail();
+
+    expect($operation->kind)->toBe(OperationKind::REGISTRY_PROVISION)
+        ->and($step->script)->toContain('registry:2')
+        ->and($step->script)->toContain('/mnt/keystone-registry')
+        ->and($step->script)->toContain('htpasswd')
+        ->and($step->script)->toContain('-Bni')
+        ->and($step->script)->not->toContain('"$build_password" > "$tmp_htpasswd"')
+        ->and($step->script)->not->toContain('set -x')
+        ->and($step->script)->toContain('REGISTRY_STORAGE_DELETE_ENABLED=true')
+        ->and($step->script)->toContain('reverse_proxy 127.0.0.1:5000')
+        ->and($step->script)->toContain('Registry proxy reload skipped')
+        ->and($step->script)->toContain('https://"$registry_host"/v2/')
+        ->and($step->script)->toContain('[!build_password_base64!]')
+        ->and($step->script)->toContain('[!runtime_password_base64!]')
+        ->and($step->script)->not->toContain($registry->credentials['build_password'])
+        ->and($step->script)->not->toContain($registry->credentials['runtime_password'])
+        ->and($step->getRawOriginal('secrets'))->not->toContain($registry->credentials['build_password'])
+        ->and($step->getRawOriginal('secrets'))->not->toContain($registry->credentials['runtime_password']);
+});
+
+it('creates managed registry smoke checks for build and runtime nodes without raw passwords', function () {
+    $organisation = Organisation::factory()->create();
+    $control = managedRegistryServerFor($organisation);
+    $runtime = managedRegistryServerFor($organisation);
+    $registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $control);
+
+    $operation = app(CreateManagedRegistrySmokeCheckOperation::class)->execute($registry, $control, [$runtime]);
+    $buildStep = $operation->steps()->firstOrFail();
+    $runtimeStep = $operation->children()->firstOrFail()->steps()->firstOrFail();
+
+    expect($operation->kind)->toBe(OperationKind::REGISTRY_HEALTH_CHECK)
+        ->and($registry->refresh()->readiness_checks)->toHaveKeys(['control_https', 'build_push', 'runtime_pull_server_'.$runtime->id])
+        ->and($buildStep->script)->toContain('https://"$registry_host"/v2/')
+        ->and($buildStep->script)->toContain('docker push "$image_ref"')
+        ->and($runtimeStep->script)->toContain('docker pull "$image_ref"')
+        ->and($buildStep->script)->not->toContain($registry->credentials['build_password'])
+        ->and($runtimeStep->script)->not->toContain($registry->credentials['runtime_password']);
+});
+
+it('marks the registry unhealthy when a runtime smoke check child operation fails', function () {
+    $organisation = Organisation::factory()->create();
+    $control = managedRegistryServerFor($organisation);
+    $runtime = managedRegistryServerFor($organisation);
+    $registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $control);
+
+    app()->instance(RemoteCommandRunner::class, new class($runtime->id) implements RemoteCommandRunner
+    {
+        public function __construct(private readonly int $runtimeServerId) {}
+
+        public function run(Server $server, string $script): string
+        {
+            if ($server->id === $this->runtimeServerId) {
+                throw new RuntimeException('runtime pull failed');
+            }
+
+            return 'ok';
+        }
+    });
+
+    $operation = app(CreateManagedRegistrySmokeCheckOperation::class)->execute($registry, $control, [$runtime]);
+    $runtimeOperation = $operation->children()->firstOrFail();
+
+    (new RunStep($runtimeOperation->steps()->firstOrFail()))->handle();
+
+    expect($registry->refresh()->health_status)->toBe('unhealthy')
+        ->and($registry->health_message)->toContain('runtime pull failed');
+});
+
+it('clears registry auth operation secrets after failed execution', function () {
+    $organisation = Organisation::factory()->create();
+    $server = managedRegistryServerFor($organisation);
+    $registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server);
+    $registry->markHealthy();
+
+    app()->instance(RemoteCommandRunner::class, new class implements RemoteCommandRunner
+    {
+        public function run(Server $server, string $script): string
+        {
+            throw new RuntimeException('failed');
+        }
+    });
+
+    $operation = app(CreateRegistryAuthOperation::class)->execute($registry, $server, 'runtime');
+    $step = $operation->steps()->firstOrFail();
+
+    (new RunStep($step))->handle();
+
+    expect($step->refresh()->status)->toBe(OperationStatus::FAILED)
+        ->and($step->secrets)->toBeNull();
+});
+
+it('marks old successful managed registry artifacts prunable without touching active or retained artifacts', function () {
+    $organisation = Organisation::factory()->create();
+    $server = managedRegistryServerFor($organisation);
+    $registry = app(ManagedRegistryProvisioner::class)->provision(
+        organisation: $organisation,
+        url: 'registry.example.com',
+        controlServer: $server,
+        retention: 3,
+    );
+    $registry->markHealthy();
+    $application = Application::factory()->for($organisation)->create();
+    $environment = Environment::factory()->for($application)->create();
+    Service::factory()->for($environment)->create([
+        'organisation_id' => $organisation->id,
+        'available_image_digest' => 'sha256:old-active',
+        'current_image_digest' => 'sha256:current-active',
+    ]);
+
+    foreach (range(1, 6) as $index) {
+        $environment->buildArtifacts()->create([
+            'commit_sha' => str_repeat((string) $index, 40),
+            'image_tag' => "keystone/{$application->uuid}/{$environment->uuid}:{$index}",
+            'registry_ref' => "registry.example.com/keystone/{$application->uuid}/{$environment->uuid}:{$index}",
+            'image_digest' => match ($index) {
+                1 => 'sha256:old-active',
+                2 => 'sha256:current-active',
+                default => 'sha256:artifact-'.$index,
+            },
+            'status' => BuildArtifactStatus::AVAILABLE,
+            'created_at' => now()->subMinutes(12 - $index),
+            'updated_at' => now()->subMinutes(12 - $index),
+        ]);
+    }
+
+    $marked = app(ManagedRegistryRetention::class)->markPrunable($registry);
+
+    expect($marked)->toHaveCount(1)
+        ->and($marked->first()->image_digest)->toBe('sha256:artifact-3')
+        ->and($marked->first()->metadata['prune_command'])->toContain('/v2/keystone/')
+        ->and($environment->buildArtifacts()->where('image_digest', 'sha256:old-active')->first()->status)->toBe(BuildArtifactStatus::AVAILABLE)
+        ->and($environment->buildArtifacts()->where('image_digest', 'sha256:current-active')->first()->status)->toBe(BuildArtifactStatus::AVAILABLE);
+});
+
+it('blocks managed registry maintenance while matching builds are active', function () {
+    $organisation = Organisation::factory()->create();
+    $server = managedRegistryServerFor($organisation);
+    $registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server);
+    $application = Application::factory()->for($organisation)->create();
+    $environment = Environment::factory()->for($application)->create();
+    $environment->buildArtifacts()->create([
+        'commit_sha' => str_repeat('a', 40),
+        'image_tag' => "keystone/{$application->uuid}/{$environment->uuid}:aaaaaaaaaaaa",
+        'registry_ref' => "registry.example.com/keystone/{$application->uuid}/{$environment->uuid}:aaaaaaaaaaaa",
+        'status' => BuildArtifactStatus::BUILDING,
+    ]);
+
+    expect(fn () => app(CreateManagedRegistryMaintenanceOperation::class)->execute($registry))
+        ->toThrow(RuntimeException::class, 'Managed registry pruning cannot run while builds are active.');
+});
+
+it('creates managed registry maintenance operations that delete manifests, run gc, and mark artifacts pruned after execution', function () {
+    $organisation = Organisation::factory()->create();
+    $server = managedRegistryServerFor($organisation);
+    $registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server, retention: 1);
+    $application = Application::factory()->for($organisation)->create();
+    $environment = Environment::factory()->for($application)->create();
+
+    foreach (range(1, 3) as $index) {
+        $environment->buildArtifacts()->create([
+            'commit_sha' => str_repeat((string) $index, 40),
+            'image_tag' => "keystone/{$application->uuid}/{$environment->uuid}:{$index}",
+            'registry_ref' => "registry.example.com/keystone/{$application->uuid}/{$environment->uuid}:{$index}",
+            'image_digest' => 'sha256:artifact-'.$index,
+            'status' => BuildArtifactStatus::AVAILABLE,
+            'created_at' => now()->subMinutes(10 - $index),
+            'updated_at' => now()->subMinutes(10 - $index),
+        ]);
+    }
+
+    app()->instance(RemoteCommandRunner::class, new class implements RemoteCommandRunner
+    {
+        public function run(Server $server, string $script): string
+        {
+            return 'ok';
+        }
+    });
+
+    $operation = app(CreateManagedRegistryMaintenanceOperation::class)->execute($registry);
+    $step = $operation->steps()->firstOrFail();
+    $unselectedArtifact = $environment->buildArtifacts()->create([
+        'commit_sha' => str_repeat('9', 40),
+        'image_tag' => "keystone/{$application->uuid}/{$environment->uuid}:9",
+        'registry_ref' => "registry.example.com/keystone/{$application->uuid}/{$environment->uuid}:9",
+        'image_digest' => 'sha256:not-in-this-maintenance-batch',
+        'status' => BuildArtifactStatus::PRUNABLE,
+    ]);
+
+    expect($step->script)->toContain('flock -n 9')
+        ->and($step->script)->toContain('--request DELETE')
+        ->and($step->script)->toContain('delete_failures=0')
+        ->and($step->script)->toContain('status=$(curl')
+        ->and($step->script)->not->toContain('--user "$username:$password"')
+        ->and($step->script)->toContain('Accept: application/vnd.docker.distribution.manifest.v2+json')
+        ->and($step->script)->toContain('garbage-collect --delete-untagged')
+        ->and($step->script)->toContain('docker stop keystone-managed-registry')
+        ->and($step->script)->toContain('docker start keystone-managed-registry')
+        ->and($step->script)->not->toContain($registry->credentials['build_password']);
+
+    (new RunStep($step))->handle();
+
+    expect($environment->buildArtifacts()->where('status', BuildArtifactStatus::PRUNED)->count())->toBe(2);
+    expect($unselectedArtifact->refresh()->status)->toBe(BuildArtifactStatus::PRUNABLE);
+});
+
+it('keeps managed registry builds blocked until represented smoke checks pass', function () {
+    $organisation = Organisation::factory()->create();
+    $server = managedRegistryServerFor($organisation);
+    $runtime = managedRegistryServerFor($organisation);
+    $registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server);
+    app(CreateManagedRegistrySmokeCheckOperation::class)->execute($registry, $server, [$runtime]);
+
+    expect(app(ManagedRegistryHealth::class)->readinessBlocker($registry->refresh()))
+        ->toBe('Managed registry has not passed readiness checks.');
+
+    app()->instance(RemoteCommandRunner::class, new class implements RemoteCommandRunner
+    {
+        public function run(Server $server, string $script): string
+        {
+            return 'ok';
+        }
+    });
+
+    $operation = $server->operations()->where('kind', OperationKind::REGISTRY_HEALTH_CHECK)->latest()->firstOrFail();
+    (new RunStep($operation->steps()->firstOrFail()))->handle();
+
+    expect($registry->refresh()->health_status)->toBe('healthy')
+        ->and($registry->ready_at)->not->toBeNull()
+        ->and(app(ManagedRegistryHealth::class)->readinessBlocker($registry))->toBeNull();
+});
+
+function managedRegistryServerFor(Organisation $organisation): Server
+{
+    $provider = Provider::factory()->forOrganisation($organisation)->create();
+    $network = Network::create([
+        'organisation_id' => $organisation->id,
+        'provider_id' => $provider->id,
+        'name' => 'test-network',
+        'ip_range' => '10.0.0.0/24',
+    ]);
+
+    return Server::factory()
+        ->forOrganisation($organisation->id)
+        ->forProvider($provider->id)
+        ->forNetwork($network->id)
+        ->create([
+            'is_control_node' => true,
+            'build_enabled' => true,
+        ]);
+}