'/mnt/registry', 'keystone.managed_registry.retention.successful_artifacts_per_environment' => 5, ]); $organisation = Organisation::factory()->create(); $server = managedRegistryServerFor($organisation); $registry = app(ManagedRegistryProvisioner::class)->provision( organisation: $organisation, url: 'https://registry.example.com', controlServer: $server, ); expect($registry->url)->toBe('registry.example.com') ->and($registry->storage_path)->toBe('/mnt/registry') ->and($registry->retention_successful_artifacts)->toBe(5) ->and($registry->control_server_id)->toBe($server->id) ->and($registry->credentials)->toHaveKeys([ 'build_username', 'build_password', 'runtime_username', 'runtime_password', ]) ->and($registry->getRawOriginal('credentials'))->not->toContain($registry->credentials['build_password']) ->and($server->refresh()->is_control_node)->toBeTrue() ->and($server->build_enabled)->toBeTrue(); }); it('checks managed registry readiness over https and records health', function () { Http::fake([ 'https://registry.example.com/v2/' => Http::response('', 401), ]); $organisation = Organisation::factory()->create(); $server = managedRegistryServerFor($organisation); $registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server); $registry->forceFill([ 'readiness_checks' => [ 'control_https' => 'pending', 'build_push' => 'passed', 'runtime_pull_server_'.$server->id => 'passed', ], ])->save(); expect(app(ManagedRegistryHealth::class)->check($registry))->toBeTrue() ->and($registry->refresh()->health_status)->toBe('healthy') ->and($registry->ready_at)->not->toBeNull(); }); it('creates registry auth operations with encrypted secrets and placeholder scripts', function () { $organisation = Organisation::factory()->create(); $server = managedRegistryServerFor($organisation); $registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server); $registry->markHealthy(); $operation = app(CreateRegistryAuthOperation::class)->execute($registry, $server, 'runtime'); $step = $operation->steps()->firstOrFail(); expect($operation->kind)->toBe(OperationKind::CREDENTIAL_ROTATION) ->and($step->script)->toContain('docker login') ->and($step->script)->toContain('[!registry_password_base64!]') ->and($step->script)->not->toContain($registry->credentials['runtime_password']) ->and($step->scriptForExecution())->toContain(base64_encode($registry->credentials['runtime_password'])) ->and($step->scriptForExecution())->not->toContain($registry->credentials['runtime_password']) ->and($step->getRawOriginal('secrets'))->not->toContain($registry->credentials['runtime_password']); }); it('creates a managed registry provision operation with registry service, htpasswd auth, storage, deletion, and caddy proxy without raw passwords', function () { $organisation = Organisation::factory()->create(); $server = managedRegistryServerFor($organisation); $registry = app(ManagedRegistryProvisioner::class)->provision( organisation: $organisation, url: 'https://registry.example.com', controlServer: $server, storagePath: '/mnt/keystone-registry', ); $operation = app(CreateManagedRegistryProvisionOperation::class)->execute($registry); $step = $operation->steps()->firstOrFail(); expect($operation->kind)->toBe(OperationKind::REGISTRY_PROVISION) ->and($step->script)->toContain('registry:2') ->and($step->script)->toContain('/mnt/keystone-registry') ->and($step->script)->toContain('htpasswd') ->and($step->script)->toContain('-Bni') ->and($step->script)->not->toContain('"$build_password" > "$tmp_htpasswd"') ->and($step->script)->not->toContain('set -x') ->and($step->script)->toContain('REGISTRY_STORAGE_DELETE_ENABLED=true') ->and($step->script)->toContain('reverse_proxy 127.0.0.1:5000') ->and($step->script)->toContain('Registry proxy reload skipped') ->and($step->script)->toContain('https://"$registry_host"/v2/') ->and($step->script)->toContain('[!build_password_base64!]') ->and($step->script)->toContain('[!runtime_password_base64!]') ->and($step->script)->not->toContain($registry->credentials['build_password']) ->and($step->script)->not->toContain($registry->credentials['runtime_password']) ->and($step->getRawOriginal('secrets'))->not->toContain($registry->credentials['build_password']) ->and($step->getRawOriginal('secrets'))->not->toContain($registry->credentials['runtime_password']); }); it('creates managed registry smoke checks for build and runtime nodes without raw passwords', function () { $organisation = Organisation::factory()->create(); $control = managedRegistryServerFor($organisation); $runtime = managedRegistryServerFor($organisation); $registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $control); $operation = app(CreateManagedRegistrySmokeCheckOperation::class)->execute($registry, $control, [$runtime]); $buildStep = $operation->steps()->firstOrFail(); $runtimeStep = $operation->children()->firstOrFail()->steps()->firstOrFail(); expect($operation->kind)->toBe(OperationKind::REGISTRY_HEALTH_CHECK) ->and($registry->refresh()->readiness_checks)->toHaveKeys(['control_https', 'build_push', 'runtime_pull_server_'.$runtime->id]) ->and($buildStep->script)->toContain('https://"$registry_host"/v2/') ->and($buildStep->script)->toContain('docker push "$image_ref"') ->and($runtimeStep->script)->toContain('docker pull "$image_ref"') ->and($buildStep->script)->not->toContain($registry->credentials['build_password']) ->and($runtimeStep->script)->not->toContain($registry->credentials['runtime_password']); }); it('marks the registry unhealthy when a runtime smoke check child operation fails', function () { $organisation = Organisation::factory()->create(); $control = managedRegistryServerFor($organisation); $runtime = managedRegistryServerFor($organisation); $registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $control); app()->instance(RemoteCommandRunner::class, new class($runtime->id) implements RemoteCommandRunner { public function __construct(private readonly int $runtimeServerId) {} public function run(Server $server, string $script): string { if ($server->id === $this->runtimeServerId) { throw new RuntimeException('runtime pull failed'); } return 'ok'; } }); $operation = app(CreateManagedRegistrySmokeCheckOperation::class)->execute($registry, $control, [$runtime]); $runtimeOperation = $operation->children()->firstOrFail(); (new RunStep($runtimeOperation->steps()->firstOrFail()))->handle(); expect($registry->refresh()->health_status)->toBe('unhealthy') ->and($registry->health_message)->toContain('runtime pull failed'); }); it('clears registry auth operation secrets after failed execution', function () { $organisation = Organisation::factory()->create(); $server = managedRegistryServerFor($organisation); $registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server); $registry->markHealthy(); app()->instance(RemoteCommandRunner::class, new class implements RemoteCommandRunner { public function run(Server $server, string $script): string { throw new RuntimeException('failed'); } }); $operation = app(CreateRegistryAuthOperation::class)->execute($registry, $server, 'runtime'); $step = $operation->steps()->firstOrFail(); (new RunStep($step))->handle(); expect($step->refresh()->status)->toBe(OperationStatus::FAILED) ->and($step->secrets)->toBeNull(); }); it('marks old successful managed registry artifacts prunable without touching active or retained artifacts', function () { $organisation = Organisation::factory()->create(); $server = managedRegistryServerFor($organisation); $registry = app(ManagedRegistryProvisioner::class)->provision( organisation: $organisation, url: 'registry.example.com', controlServer: $server, retention: 3, ); $registry->markHealthy(); $application = Application::factory()->for($organisation)->create(); $environment = Environment::factory()->for($application)->create(); Service::factory()->for($environment)->create([ 'organisation_id' => $organisation->id, 'available_image_digest' => 'sha256:old-active', 'current_image_digest' => 'sha256:current-active', ]); foreach (range(1, 6) as $index) { $environment->buildArtifacts()->create([ 'commit_sha' => str_repeat((string) $index, 40), 'image_tag' => "keystone/{$application->uuid}/{$environment->uuid}:{$index}", 'registry_ref' => "registry.example.com/keystone/{$application->uuid}/{$environment->uuid}:{$index}", 'image_digest' => match ($index) { 1 => 'sha256:old-active', 2 => 'sha256:current-active', default => 'sha256:artifact-'.$index, }, 'status' => BuildArtifactStatus::AVAILABLE, 'created_at' => now()->subMinutes(12 - $index), 'updated_at' => now()->subMinutes(12 - $index), ]); } $marked = app(ManagedRegistryRetention::class)->markPrunable($registry); expect($marked)->toHaveCount(1) ->and($marked->first()->image_digest)->toBe('sha256:artifact-3') ->and($marked->first()->metadata['prune_command'])->toContain('/v2/keystone/') ->and($environment->buildArtifacts()->where('image_digest', 'sha256:old-active')->first()->status)->toBe(BuildArtifactStatus::AVAILABLE) ->and($environment->buildArtifacts()->where('image_digest', 'sha256:current-active')->first()->status)->toBe(BuildArtifactStatus::AVAILABLE); }); it('blocks managed registry maintenance while matching builds are active', function () { $organisation = Organisation::factory()->create(); $server = managedRegistryServerFor($organisation); $registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server); $application = Application::factory()->for($organisation)->create(); $environment = Environment::factory()->for($application)->create(); $environment->buildArtifacts()->create([ 'commit_sha' => str_repeat('a', 40), 'image_tag' => "keystone/{$application->uuid}/{$environment->uuid}:aaaaaaaaaaaa", 'registry_ref' => "registry.example.com/keystone/{$application->uuid}/{$environment->uuid}:aaaaaaaaaaaa", 'status' => BuildArtifactStatus::BUILDING, ]); expect(fn () => app(CreateManagedRegistryMaintenanceOperation::class)->execute($registry)) ->toThrow(RuntimeException::class, 'Managed registry pruning cannot run while builds are active.'); }); it('creates managed registry maintenance operations that delete manifests, run gc, and mark artifacts pruned after execution', function () { $organisation = Organisation::factory()->create(); $server = managedRegistryServerFor($organisation); $registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server, retention: 1); $application = Application::factory()->for($organisation)->create(); $environment = Environment::factory()->for($application)->create(); foreach (range(1, 3) as $index) { $environment->buildArtifacts()->create([ 'commit_sha' => str_repeat((string) $index, 40), 'image_tag' => "keystone/{$application->uuid}/{$environment->uuid}:{$index}", 'registry_ref' => "registry.example.com/keystone/{$application->uuid}/{$environment->uuid}:{$index}", 'image_digest' => 'sha256:artifact-'.$index, 'status' => BuildArtifactStatus::AVAILABLE, 'created_at' => now()->subMinutes(10 - $index), 'updated_at' => now()->subMinutes(10 - $index), ]); } app()->instance(RemoteCommandRunner::class, new class implements RemoteCommandRunner { public function run(Server $server, string $script): string { return 'ok'; } }); $operation = app(CreateManagedRegistryMaintenanceOperation::class)->execute($registry); $step = $operation->steps()->firstOrFail(); $unselectedArtifact = $environment->buildArtifacts()->create([ 'commit_sha' => str_repeat('9', 40), 'image_tag' => "keystone/{$application->uuid}/{$environment->uuid}:9", 'registry_ref' => "registry.example.com/keystone/{$application->uuid}/{$environment->uuid}:9", 'image_digest' => 'sha256:not-in-this-maintenance-batch', 'status' => BuildArtifactStatus::PRUNABLE, ]); expect($step->script)->toContain('flock -n 9') ->and($step->script)->toContain('--request DELETE') ->and($step->script)->toContain('delete_failures=0') ->and($step->script)->toContain('status=$(curl') ->and($step->script)->not->toContain('--user "$username:$password"') ->and($step->script)->toContain('Accept: application/vnd.docker.distribution.manifest.v2+json') ->and($step->script)->toContain('garbage-collect --delete-untagged') ->and($step->script)->toContain('docker stop keystone-managed-registry') ->and($step->script)->toContain('docker start keystone-managed-registry') ->and($step->script)->not->toContain($registry->credentials['build_password']); (new RunStep($step))->handle(); expect($environment->buildArtifacts()->where('status', BuildArtifactStatus::PRUNED)->count())->toBe(2); expect($unselectedArtifact->refresh()->status)->toBe(BuildArtifactStatus::PRUNABLE); }); it('keeps managed registry builds blocked until represented smoke checks pass', function () { $organisation = Organisation::factory()->create(); $server = managedRegistryServerFor($organisation); $runtime = managedRegistryServerFor($organisation); $registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server); app(CreateManagedRegistrySmokeCheckOperation::class)->execute($registry, $server, [$runtime]); expect(app(ManagedRegistryHealth::class)->readinessBlocker($registry->refresh())) ->toBe('Managed registry has not passed readiness checks.'); app()->instance(RemoteCommandRunner::class, new class implements RemoteCommandRunner { public function run(Server $server, string $script): string { return 'ok'; } }); $operation = $server->operations()->where('kind', OperationKind::REGISTRY_HEALTH_CHECK)->latest()->firstOrFail(); (new RunStep($operation->steps()->firstOrFail()))->handle(); expect($registry->refresh()->health_status)->toBe('healthy') ->and($registry->ready_at)->not->toBeNull() ->and(app(ManagedRegistryHealth::class)->readinessBlocker($registry))->toBeNull(); }); function managedRegistryServerFor(Organisation $organisation): Server { $provider = Provider::factory()->forOrganisation($organisation)->create(); $network = Network::create([ 'organisation_id' => $organisation->id, 'provider_id' => $provider->id, 'name' => 'test-network', 'ip_range' => '10.0.0.0/24', ]); return Server::factory() ->forOrganisation($organisation->id) ->forProvider($provider->id) ->forNetwork($network->id) ->create([ 'is_control_node' => true, 'build_enabled' => true, ]); }