Add managed registry provisioning, pruning, and readiness tracking

This commit is contained in:
2026-06-08 20:44:16 +01:00
parent 5b977c1f41
commit 3a851db08f
52 changed files with 2706 additions and 116 deletions

View File

@@ -5,6 +5,7 @@ use App\Actions\Applications\GenerateDeployKey;
use App\Actions\Environments\BuildApplicationArtifact;
use App\Actions\Environments\PlanBuildArtifact;
use App\Enums\BuildArtifactStatus;
use App\Enums\BuildStrategy;
use App\Enums\RegistryType;
use App\Models\Application;
use App\Models\Network;
@@ -23,7 +24,7 @@ beforeEach(function () {
{
$this->scripts[] = $script;
return str_contains($script, 'docker manifest inspect')
return str_contains($script, 'docker buildx imagetools inspect') || str_contains($script, 'push_output=$(docker push')
? "image_digest=sha256:registrydigest\n"
: "image_digest=billing-api:aaaaaaaaaaaa@sha256:localdigest\n";
}
@@ -80,13 +81,64 @@ it('resolves external registry artifacts without building locally', function ()
expect($built->registry_ref)->toBe('ghcr.io/example/billing-api:bbbbbbbbbbbb')
->and($built->image_digest)->toBe('sha256:registrydigest')
->and($this->remoteRunner->scripts[0])->toContain('docker manifest inspect')
->and($this->remoteRunner->scripts[0])->toContain('docker buildx imagetools inspect')
->and($this->remoteRunner->scripts[0])->toContain('ghcr.io/example/billing-api:bbbbbbbbbbbb')
->and($this->remoteRunner->scripts[0])->not->toContain('docker build')
->and($this->remoteRunner->scripts[0])->not->toContain('docker build --file')
->and($this->remoteRunner->scripts[0])->not->toContain('git clone');
});
function buildServerFor(Organisation $organisation): Server
it('builds and pushes managed registry artifacts without embedding registry credentials', function () {
$organisation = Organisation::factory()->create();
$server = buildServerFor($organisation, true);
$organisation->registries()->create([
'name' => 'Managed',
'type' => RegistryType::MANAGED,
'url' => 'registry.example.com',
'credentials' => [
'build_username' => 'keystone-build',
'build_password' => 'super-secret-password',
'runtime_username' => 'keystone-runtime',
'runtime_password' => 'runtime-secret',
],
'control_server_id' => $server->id,
'health_status' => 'healthy',
'readiness_checks' => ['control_https' => 'passed', 'build_push' => 'passed'],
'ready_at' => now(),
]);
$application = Application::factory()->for($organisation)->create([
'name' => 'Billing API',
'repository_url' => 'git@example.com:org/repo.git',
]);
app(GenerateDeployKey::class)->execute($application, [
'public' => 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITestPublicKey keystone',
'private' => "-----BEGIN OPENSSH PRIVATE KEY-----\ntest\n-----END OPENSSH PRIVATE KEY-----",
'fingerprint' => 'SHA256:test',
]);
$environment = app(CreateLaravelEnvironment::class)->execute($application->refresh(), 'production');
$environment->services()->first()->update([
'server_id' => $server->id,
'desired_replicas' => 2,
]);
$artifact = app(PlanBuildArtifact::class)->execute($environment, str_repeat('c', 40));
$built = app(BuildApplicationArtifact::class)->execute($artifact);
expect($built->registry_ref)->toBe("registry.example.com/keystone/{$application->uuid}/{$environment->uuid}:cccccccccccc")
->and($built->metadata['build_strategy'])->toBe(BuildStrategy::DEDICATED_BUILDER->value)
->and($built->image_digest)->toBe('sha256:registrydigest')
->and($this->remoteRunner->scripts[0])->toContain('flock 9')
->and($this->remoteRunner->scripts[0])->toContain('docker login')
->and($this->remoteRunner->scripts[0])->toContain(base64_encode('super-secret-password'))
->and($this->remoteRunner->scripts[0])->toContain('docker build --file Dockerfile.keystone')
->and($this->remoteRunner->scripts[0])->toContain('push_output=$(docker push')
->and($this->remoteRunner->scripts[0])->toContain('digest: \\(sha256:')
->and($this->remoteRunner->scripts[0])->not->toContain('docker manifest inspect')
->and($this->remoteRunner->scripts[0])->not->toContain('"digest"')
->and($this->remoteRunner->scripts[0])->not->toContain('super-secret-password')
->and($this->remoteRunner->scripts[0])->toContain('DOCKER_CONFIG=\'/root/.docker\'');
});
function buildServerFor(Organisation $organisation, bool $buildEnabled = false): Server
{
$provider = Provider::factory()->forOrganisation($organisation)->create();
$network = Network::create([
@@ -100,5 +152,8 @@ function buildServerFor(Organisation $organisation): Server
->forOrganisation($organisation->id)
->forProvider($provider->id)
->forNetwork($network->id)
->create();
->create([
'is_control_node' => $buildEnabled,
'build_enabled' => $buildEnabled,
]);
}

View File

@@ -9,7 +9,10 @@ use App\Enums\ServiceCategory;
use App\Enums\ServiceType;
use App\Models\Application;
use App\Models\Environment;
use App\Models\Network;
use App\Models\Organisation;
use App\Models\Provider;
use App\Models\Server;
use App\Models\Service;
it('plans single-server builds on the target server without requiring a registry', function () {
@@ -36,6 +39,8 @@ it('plans single-server builds on the target server without requiring a registry
});
it('requires a registry before planning multi-server builds', function () {
config(['keystone.managed_registry.url' => null]);
$organisation = Organisation::factory()->create();
$application = Application::factory()->for($organisation)->create();
$environment = Environment::factory()->for($application)->create();
@@ -54,6 +59,100 @@ it('requires a registry before planning multi-server builds', function () {
->toThrow(RuntimeException::class, 'A registry is required before building artifacts for multi-server deployments.');
});
it('plans multi-server builds against a persisted managed registry', function () {
$organisation = Organisation::factory()->create();
$server = buildEnabledServerFor($organisation);
$organisation->registries()->create([
'name' => 'Managed',
'type' => RegistryType::MANAGED,
'url' => 'registry.example.com',
'credentials' => [
'build_username' => 'keystone-build',
'build_password' => 'secret',
'runtime_username' => 'keystone-runtime',
'runtime_password' => 'runtime-secret',
],
'control_server_id' => $server->id,
'health_status' => 'healthy',
'readiness_checks' => ['control_https' => 'passed', 'build_push' => 'passed'],
'ready_at' => now(),
]);
$application = Application::factory()->for($organisation)->create(['name' => 'Billing API']);
$environment = Environment::factory()->for($application)->create();
Service::factory()->for($environment)->create([
'organisation_id' => $organisation->id,
'category' => ServiceCategory::APPLICATION,
'type' => ServiceType::LARAVEL,
'version' => 'php-8.4',
'version_track' => 'php-8.4',
'driver_name' => 'laravel.php-8.4',
'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT,
'desired_replicas' => 2,
]);
$artifact = app(PlanBuildArtifact::class)->execute($environment, str_repeat('d', 40));
expect($artifact->image_tag)->toBe("keystone/{$application->uuid}/{$environment->uuid}:dddddddddddd")
->and($artifact->registry_ref)->toBe("registry.example.com/keystone/{$application->uuid}/{$environment->uuid}:dddddddddddd")
->and($artifact->metadata['build_strategy'])->toBe(BuildStrategy::DEDICATED_BUILDER->value)
->and($artifact->metadata['build_server_id'])->toBe($server->id)
->and($artifact->metadata['registry_type'])->toBe(RegistryType::MANAGED->value);
});
it('blocks managed registry builds until the registry is ready', function () {
$organisation = Organisation::factory()->create();
$server = buildEnabledServerFor($organisation);
$organisation->registries()->create([
'name' => 'Managed',
'type' => RegistryType::MANAGED,
'url' => 'registry.example.com',
'credentials' => [
'build_username' => 'keystone-build',
'build_password' => 'secret',
'runtime_username' => 'keystone-runtime',
'runtime_password' => 'runtime-secret',
],
'control_server_id' => $server->id,
'health_status' => 'pending',
]);
$application = Application::factory()->for($organisation)->create();
$environment = Environment::factory()->for($application)->create();
Service::factory()->for($environment)->create([
'organisation_id' => $organisation->id,
'category' => ServiceCategory::APPLICATION,
'type' => ServiceType::LARAVEL,
'version' => 'php-8.4',
'version_track' => 'php-8.4',
'driver_name' => 'laravel.php-8.4',
'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT,
'desired_replicas' => 2,
]);
expect(fn () => app(PlanBuildArtifact::class)->execute($environment, str_repeat('g', 40)))
->toThrow(RuntimeException::class, 'Managed registry has not passed readiness checks.');
});
it('does not treat the managed registry config value as a ready registry record', function () {
config(['keystone.managed_registry.url' => 'registry.example.com']);
$organisation = Organisation::factory()->create();
$application = Application::factory()->for($organisation)->create();
$environment = Environment::factory()->for($application)->create();
Service::factory()->for($environment)->create([
'organisation_id' => $organisation->id,
'category' => ServiceCategory::APPLICATION,
'type' => ServiceType::LARAVEL,
'version' => 'php-8.4',
'version_track' => 'php-8.4',
'driver_name' => 'laravel.php-8.4',
'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT,
'desired_replicas' => 2,
]);
expect(fn () => app(PlanBuildArtifact::class)->execute($environment, str_repeat('e', 40)))
->toThrow(RuntimeException::class, 'A registry is required before building artifacts for multi-server deployments.');
});
it('plans multi-server builds against the configured external registry', function () {
$organisation = Organisation::factory()->create();
$organisation->registries()->create([
@@ -79,3 +178,66 @@ it('plans multi-server builds against the configured external registry', functio
expect($artifact->registry_ref)->toBe('ghcr.io/example/billing-api:cccccccccccc')
->and($artifact->metadata['build_strategy'])->toBe(BuildStrategy::EXTERNAL_REGISTRY->value);
});
it('prefers a configured external registry over the managed default', function () {
$organisation = Organisation::factory()->create();
$server = buildEnabledServerFor($organisation);
$organisation->registries()->create([
'name' => 'Managed',
'type' => RegistryType::MANAGED,
'url' => 'registry.example.com',
'credentials' => [
'build_username' => 'keystone-build',
'build_password' => 'secret',
'runtime_username' => 'keystone-runtime',
'runtime_password' => 'runtime-secret',
],
'control_server_id' => $server->id,
'health_status' => 'healthy',
'readiness_checks' => ['control_https' => 'passed', 'build_push' => 'passed'],
'ready_at' => now(),
]);
$organisation->registries()->create([
'name' => 'GHCR',
'type' => RegistryType::GHCR,
'url' => 'ghcr.io/example',
]);
$application = Application::factory()->for($organisation)->create(['name' => 'Billing API']);
$environment = Environment::factory()->for($application)->create();
Service::factory()->for($environment)->create([
'organisation_id' => $organisation->id,
'category' => ServiceCategory::APPLICATION,
'type' => ServiceType::LARAVEL,
'version' => 'php-8.4',
'version_track' => 'php-8.4',
'driver_name' => 'laravel.php-8.4',
'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT,
'desired_replicas' => 2,
]);
$artifact = app(PlanBuildArtifact::class)->execute($environment, str_repeat('f', 40));
expect($artifact->registry_ref)->toBe('ghcr.io/example/billing-api:ffffffffffff')
->and($artifact->metadata['build_strategy'])->toBe(BuildStrategy::EXTERNAL_REGISTRY->value)
->and($artifact->metadata['registry_type'])->toBe(RegistryType::GHCR->value);
});
function buildEnabledServerFor(Organisation $organisation): Server
{
$provider = Provider::factory()->forOrganisation($organisation)->create();
$network = Network::create([
'organisation_id' => $organisation->id,
'provider_id' => $provider->id,
'name' => 'test-network',
'ip_range' => '10.0.0.0/24',
]);
return Server::factory()
->forOrganisation($organisation->id)
->forProvider($provider->id)
->forNetwork($network->id)
->create([
'is_control_node' => true,
'build_enabled' => true,
]);
}

View File

@@ -57,7 +57,7 @@ it('creates a parent environment operation with child service deploy operations'
->forProvider($provider->id)
->forNetwork($network->id)
->create();
$application = Application::factory()->for($organisation)->create();
$application = Application::factory()->for($organisation)->create(['name' => 'Keystone Test App']);
generateDeployKey($application);
$environment = Environment::factory()->for($application)->create();
$service = Service::factory()->for($environment)->for($server)->create([
@@ -113,7 +113,7 @@ it('creates replica route configure and gateway cutover child operations', funct
->forProvider($provider->id)
->forNetwork($network->id)
->create();
$application = Application::factory()->for($organisation)->create();
$application = Application::factory()->for($organisation)->create(['name' => 'Keystone Test App']);
generateDeployKey($application);
$environment = Environment::factory()->for($application)->create();
$web = Service::factory()->for($environment)->for($server)->create([
@@ -191,6 +191,7 @@ it('creates replica route configure and gateway cutover child operations', funct
->first();
$serviceDeploy = $parent->children()->where('kind', OperationKind::SERVICE_DEPLOY)->first();
$compose = renderedComposeFrom($serviceDeploy->steps()->where('name', 'Render Compose files')->first()->script, $web->id);
expect($serviceDeploy->children()->where('kind', OperationKind::REPLICA_DEPLOY)->count())->toBe(2)
->and($parent->children()->where('kind', OperationKind::SLICE_CONFIGURE)->count())->toBe(1)
@@ -203,6 +204,8 @@ it('creates replica route configure and gateway cutover child operations', funct
->toContain('docker pull')
->and($serviceDeploy->children()->where('kind', OperationKind::REPLICA_DEPLOY)->first()->steps()->where('name', 'Pull image for replica 1')->first()->script)
->toContain('@sha256:deploymentdigest')
->and($compose)
->toContain('image: "registry.example.com/keystone-test-app:aaaaaaaaaaaa@sha256:deploymentdigest"')
->and($serviceDeploy->children()->where('kind', OperationKind::REPLICA_DEPLOY)->first()->steps()->where('name', 'Health check replica 1')->first()->script)
->toContain('health_status=')
->and($parent->children()->where('kind', OperationKind::SLICE_CONFIGURE)->first()->steps()->first()->script)
@@ -364,6 +367,80 @@ it('places desired replicas across configured server placements', function () {
->toBe($servers->pluck('id')->all());
});
it('renders compose and root registry auth on each managed registry replica server', function () {
$organisation = Organisation::factory()->create();
$provider = Provider::factory()->forOrganisation($organisation)->create();
$network = Network::create([
'organisation_id' => $organisation->id,
'provider_id' => $provider->id,
'name' => 'test-network',
'ip_range' => '10.0.0.0/24',
]);
$servers = Server::factory()
->count(2)
->forOrganisation($organisation->id)
->forProvider($provider->id)
->forNetwork($network->id)
->create();
$servers[0]->update([
'is_control_node' => true,
'build_enabled' => true,
]);
$organisation->registries()->create([
'name' => 'Managed',
'type' => 'managed',
'url' => 'registry.example.com',
'credentials' => [
'build_username' => 'keystone-build',
'build_password' => 'build-secret',
'runtime_username' => 'keystone-runtime',
'runtime_password' => 'runtime-secret',
],
'control_server_id' => $servers[0]->id,
'health_status' => 'healthy',
'readiness_checks' => ['control_https' => 'passed', 'build_push' => 'passed'],
'ready_at' => now(),
]);
$application = Application::factory()->for($organisation)->create(['name' => 'Keystone Test App']);
generateDeployKey($application);
$environment = Environment::factory()->for($application)->create();
$service = Service::factory()->for($environment)->create([
'organisation_id' => $organisation->id,
'name' => 'web',
'category' => ServiceCategory::APPLICATION,
'type' => ServiceType::LARAVEL,
'version' => 'php-8.4',
'version_track' => 'php-8.4',
'driver_name' => 'laravel.php-8.4',
'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT,
'desired_replicas' => 2,
'config' => [
'server_ids' => $servers->pluck('id')->all(),
],
]);
(new DeployEnvironment($environment))->handle();
$serviceDeploy = $service->operations()
->where('kind', OperationKind::SERVICE_DEPLOY)
->firstOrFail();
$replicaOperations = $serviceDeploy->children()
->where('kind', OperationKind::REPLICA_DEPLOY)
->with('steps')
->get();
expect($replicaOperations)->toHaveCount(2)
->and($replicaOperations[0]->steps->pluck('name')->all())->toContain('Render replica 1 Compose files')
->and($replicaOperations[1]->steps->pluck('name')->all())->toContain('Render replica 2 Compose files');
$authStep = $replicaOperations[0]->steps->firstWhere('name', 'Configure registry auth for replica 1');
expect($authStep->script)->toContain("DOCKER_CONFIG='/root/.docker'")
->and($authStep->script)->toContain('[!registry_password_base64!]')
->and($authStep->script)->not->toContain('runtime-secret')
->and($authStep->secrets['registry_password_base64'])->toBe(base64_encode('runtime-secret'));
});
it('skips environment service operations when the target revision is already available', function () {
$organisation = Organisation::factory()->create();
$application = Application::factory()->for($organisation)->create();
@@ -404,6 +481,17 @@ function generateDeployKey(Application $application): void
]);
}
function renderedComposeFrom(string $script, int $serviceId): string
{
preg_match(
'/printf %s \'(?<encoded>[^\']+)\' \| base64 -d > \/home\/keystone\/services\/'.$serviceId.'\/compose\.yml/',
$script,
$matches,
);
return base64_decode($matches['encoded'] ?? '', true) ?: '';
}
it('blocks multi-server deploys that do not have a registry', function () {
$organisation = Organisation::factory()->create();
$application = Application::factory()->for($organisation)->create();

View File

@@ -99,6 +99,8 @@ it('runs an environment deployment from the application surface', function () {
});
it('blocks multi-server environment deployment until a registry is configured', function () {
config(['keystone.managed_registry.url' => null]);
$user = User::factory()->create();
$organisation = Organisation::factory()->create(['owner_id' => $user->id]);
$provider = Provider::factory()->forOrganisation($organisation)->create();
@@ -152,6 +154,64 @@ it('blocks multi-server environment deployment until a registry is configured',
Bus::assertDispatched(DeployEnvironment::class);
});
it('dispatches multi-server environment deployments when a managed registry exists', function () {
$user = User::factory()->create();
$organisation = Organisation::factory()->create(['owner_id' => $user->id]);
$provider = Provider::factory()->forOrganisation($organisation)->create();
$network = Network::create([
'organisation_id' => $organisation->id,
'provider_id' => $provider->id,
'name' => 'test-network',
'ip_range' => '10.0.0.0/24',
]);
$primaryServer = Server::factory()
->forOrganisation($organisation->id)
->forProvider($provider->id)
->forNetwork($network->id)
->create([
'is_control_node' => true,
'build_enabled' => true,
]);
$organisation->registries()->create([
'name' => 'Managed',
'type' => RegistryType::MANAGED,
'url' => 'registry.example.com',
'credentials' => [
'build_username' => 'keystone-build',
'build_password' => 'secret',
'runtime_username' => 'keystone-runtime',
'runtime_password' => 'runtime-secret',
],
'control_server_id' => $primaryServer->id,
'health_status' => 'healthy',
'readiness_checks' => ['control_https' => 'passed', 'build_push' => 'passed'],
'ready_at' => now(),
]);
$secondaryServer = Server::factory()
->forOrganisation($organisation->id)
->forProvider($provider->id)
->forNetwork($network->id)
->create();
$application = Application::factory()->for($organisation)->create();
$environment = Environment::factory()->for($application)->create();
$service = Service::factory()->for($environment)->for($primaryServer)->create([
'organisation_id' => $organisation->id,
'category' => ServiceCategory::APPLICATION,
'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT,
]);
ServiceReplica::factory()
->for($service)
->for($secondaryServer, 'server')
->create();
Bus::fake();
$this->actingAs($user)
->post(route('environment-deployments.store', [$organisation, $application, $environment]))
->assertRedirect(route('environments.show', [$organisation, $application, $environment]));
Bus::assertDispatched(DeployEnvironment::class);
});
it('deploys an environment at a specific commit when provided', function () {
$user = User::factory()->create();
$organisation = Organisation::factory()->create(['owner_id' => $user->id]);

View File

@@ -3,12 +3,16 @@
use App\Actions\Environments\PlanEnvironmentDeployment;
use App\Enums\DeployPolicy;
use App\Enums\EnvironmentAttachmentRole;
use App\Enums\RegistryType;
use App\Enums\SchedulerMode;
use App\Enums\ServiceCategory;
use App\Enums\ServiceType;
use App\Models\Application;
use App\Models\Environment;
use App\Models\Network;
use App\Models\Organisation;
use App\Models\Provider;
use App\Models\Server;
use App\Models\Service;
it('deploys only with-environment services and checks dependency attachments', function () {
@@ -47,6 +51,8 @@ it('deploys only with-environment services and checks dependency attachments', f
});
it('blocks multi-server environment deployments when no registry exists', function () {
config(['keystone.managed_registry.url' => null]);
$organisation = Organisation::factory()->create();
$application = Application::factory()->for($organisation)->create();
$environment = Environment::factory()->for($application)->create();
@@ -68,6 +74,102 @@ it('blocks multi-server environment deployments when no registry exists', functi
expect($plan->requiresRegistry)->toBeTrue();
});
it('blocks multi-server environment deployments when managed registry smoke checks have not passed', function () {
$organisation = Organisation::factory()->create();
$server = deploymentPlanBuildServerFor($organisation);
$organisation->registries()->create([
'name' => 'Managed',
'type' => RegistryType::MANAGED,
'url' => 'registry.example.com',
'credentials' => [
'build_username' => 'keystone-build',
'build_password' => 'secret',
'runtime_username' => 'keystone-runtime',
'runtime_password' => 'runtime-secret',
],
'control_server_id' => $server->id,
'health_status' => 'pending',
'readiness_checks' => ['control_https' => 'passed', 'build_push' => 'pending'],
]);
$application = Application::factory()->for($organisation)->create();
$environment = Environment::factory()->for($application)->create();
Service::factory()->for($environment)->create([
'organisation_id' => $organisation->id,
'name' => 'web',
'category' => ServiceCategory::APPLICATION,
'type' => ServiceType::LARAVEL,
'version' => 'php-8.4',
'version_track' => 'php-8.4',
'driver_name' => 'laravel.php-8.4',
'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT,
'desired_replicas' => 2,
]);
$plan = app(PlanEnvironmentDeployment::class)->execute($environment);
expect($plan->requiresRegistry)->toBeTrue()
->and($plan->blockers)->toContain('Managed registry has not passed readiness checks.');
});
it('allows multi-server environment deployments when a managed registry exists', function () {
$organisation = Organisation::factory()->create();
$server = deploymentPlanBuildServerFor($organisation);
$organisation->registries()->create([
'name' => 'Managed',
'type' => RegistryType::MANAGED,
'url' => 'registry.example.com',
'credentials' => [
'build_username' => 'keystone-build',
'build_password' => 'secret',
'runtime_username' => 'keystone-runtime',
'runtime_password' => 'runtime-secret',
],
'control_server_id' => $server->id,
'health_status' => 'healthy',
'readiness_checks' => ['control_https' => 'passed', 'build_push' => 'passed'],
'ready_at' => now(),
]);
$application = Application::factory()->for($organisation)->create();
$environment = Environment::factory()->for($application)->create();
Service::factory()->for($environment)->create([
'organisation_id' => $organisation->id,
'name' => 'web',
'category' => ServiceCategory::APPLICATION,
'type' => ServiceType::LARAVEL,
'version' => 'php-8.4',
'version_track' => 'php-8.4',
'driver_name' => 'laravel.php-8.4',
'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT,
'desired_replicas' => 2,
]);
$plan = app(PlanEnvironmentDeployment::class)->execute($environment);
expect($plan->requiresRegistry)->toBeFalse();
});
function deploymentPlanBuildServerFor(Organisation $organisation): Server
{
$provider = Provider::factory()->forOrganisation($organisation)->create();
$network = Network::create([
'organisation_id' => $organisation->id,
'provider_id' => $provider->id,
'name' => 'test-network',
'ip_range' => '10.0.0.0/24',
]);
return Server::factory()
->forOrganisation($organisation->id)
->forProvider($provider->id)
->forNetwork($network->id)
->create([
'is_control_node' => true,
'build_enabled' => true,
]);
}
it('warns about sync queues without creating worker services', function () {
$organisation = Organisation::factory()->create();
$application = Application::factory()->for($organisation)->create();

View File

@@ -0,0 +1,354 @@
<?php
use App\Actions\Registries\CreateManagedRegistryMaintenanceOperation;
use App\Actions\Registries\CreateManagedRegistryProvisionOperation;
use App\Actions\Registries\CreateManagedRegistrySmokeCheckOperation;
use App\Actions\Registries\CreateRegistryAuthOperation;
use App\Enums\BuildArtifactStatus;
use App\Enums\OperationKind;
use App\Enums\OperationStatus;
use App\Jobs\Services\RunStep;
use App\Models\Application;
use App\Models\Environment;
use App\Models\Network;
use App\Models\Organisation;
use App\Models\Provider;
use App\Models\Server;
use App\Models\Service;
use App\Services\Operations\RemoteCommandRunner;
use App\Services\Registries\ManagedRegistryHealth;
use App\Services\Registries\ManagedRegistryProvisioner;
use App\Services\Registries\ManagedRegistryRetention;
use Illuminate\Support\Facades\Http;
it('provisions a persisted managed registry with encrypted scoped credentials and defaults', function () {
config([
'keystone.managed_registry.storage_path' => '/mnt/registry',
'keystone.managed_registry.retention.successful_artifacts_per_environment' => 5,
]);
$organisation = Organisation::factory()->create();
$server = managedRegistryServerFor($organisation);
$registry = app(ManagedRegistryProvisioner::class)->provision(
organisation: $organisation,
url: 'https://registry.example.com',
controlServer: $server,
);
expect($registry->url)->toBe('registry.example.com')
->and($registry->storage_path)->toBe('/mnt/registry')
->and($registry->retention_successful_artifacts)->toBe(5)
->and($registry->control_server_id)->toBe($server->id)
->and($registry->credentials)->toHaveKeys([
'build_username',
'build_password',
'runtime_username',
'runtime_password',
])
->and($registry->getRawOriginal('credentials'))->not->toContain($registry->credentials['build_password'])
->and($server->refresh()->is_control_node)->toBeTrue()
->and($server->build_enabled)->toBeTrue();
});
it('checks managed registry readiness over https and records health', function () {
Http::fake([
'https://registry.example.com/v2/' => Http::response('', 401),
]);
$organisation = Organisation::factory()->create();
$server = managedRegistryServerFor($organisation);
$registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server);
$registry->forceFill([
'readiness_checks' => [
'control_https' => 'pending',
'build_push' => 'passed',
'runtime_pull_server_'.$server->id => 'passed',
],
])->save();
expect(app(ManagedRegistryHealth::class)->check($registry))->toBeTrue()
->and($registry->refresh()->health_status)->toBe('healthy')
->and($registry->ready_at)->not->toBeNull();
});
it('creates registry auth operations with encrypted secrets and placeholder scripts', function () {
$organisation = Organisation::factory()->create();
$server = managedRegistryServerFor($organisation);
$registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server);
$registry->markHealthy();
$operation = app(CreateRegistryAuthOperation::class)->execute($registry, $server, 'runtime');
$step = $operation->steps()->firstOrFail();
expect($operation->kind)->toBe(OperationKind::CREDENTIAL_ROTATION)
->and($step->script)->toContain('docker login')
->and($step->script)->toContain('[!registry_password_base64!]')
->and($step->script)->not->toContain($registry->credentials['runtime_password'])
->and($step->scriptForExecution())->toContain(base64_encode($registry->credentials['runtime_password']))
->and($step->scriptForExecution())->not->toContain($registry->credentials['runtime_password'])
->and($step->getRawOriginal('secrets'))->not->toContain($registry->credentials['runtime_password']);
});
it('creates a managed registry provision operation with registry service, htpasswd auth, storage, deletion, and caddy proxy without raw passwords', function () {
$organisation = Organisation::factory()->create();
$server = managedRegistryServerFor($organisation);
$registry = app(ManagedRegistryProvisioner::class)->provision(
organisation: $organisation,
url: 'https://registry.example.com',
controlServer: $server,
storagePath: '/mnt/keystone-registry',
);
$operation = app(CreateManagedRegistryProvisionOperation::class)->execute($registry);
$step = $operation->steps()->firstOrFail();
expect($operation->kind)->toBe(OperationKind::REGISTRY_PROVISION)
->and($step->script)->toContain('registry:2')
->and($step->script)->toContain('/mnt/keystone-registry')
->and($step->script)->toContain('htpasswd')
->and($step->script)->toContain('-Bni')
->and($step->script)->not->toContain('"$build_password" > "$tmp_htpasswd"')
->and($step->script)->not->toContain('set -x')
->and($step->script)->toContain('REGISTRY_STORAGE_DELETE_ENABLED=true')
->and($step->script)->toContain('reverse_proxy 127.0.0.1:5000')
->and($step->script)->toContain('Registry proxy reload skipped')
->and($step->script)->toContain('https://"$registry_host"/v2/')
->and($step->script)->toContain('[!build_password_base64!]')
->and($step->script)->toContain('[!runtime_password_base64!]')
->and($step->script)->not->toContain($registry->credentials['build_password'])
->and($step->script)->not->toContain($registry->credentials['runtime_password'])
->and($step->getRawOriginal('secrets'))->not->toContain($registry->credentials['build_password'])
->and($step->getRawOriginal('secrets'))->not->toContain($registry->credentials['runtime_password']);
});
it('creates managed registry smoke checks for build and runtime nodes without raw passwords', function () {
$organisation = Organisation::factory()->create();
$control = managedRegistryServerFor($organisation);
$runtime = managedRegistryServerFor($organisation);
$registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $control);
$operation = app(CreateManagedRegistrySmokeCheckOperation::class)->execute($registry, $control, [$runtime]);
$buildStep = $operation->steps()->firstOrFail();
$runtimeStep = $operation->children()->firstOrFail()->steps()->firstOrFail();
expect($operation->kind)->toBe(OperationKind::REGISTRY_HEALTH_CHECK)
->and($registry->refresh()->readiness_checks)->toHaveKeys(['control_https', 'build_push', 'runtime_pull_server_'.$runtime->id])
->and($buildStep->script)->toContain('https://"$registry_host"/v2/')
->and($buildStep->script)->toContain('docker push "$image_ref"')
->and($runtimeStep->script)->toContain('docker pull "$image_ref"')
->and($buildStep->script)->not->toContain($registry->credentials['build_password'])
->and($runtimeStep->script)->not->toContain($registry->credentials['runtime_password']);
});
it('marks the registry unhealthy when a runtime smoke check child operation fails', function () {
$organisation = Organisation::factory()->create();
$control = managedRegistryServerFor($organisation);
$runtime = managedRegistryServerFor($organisation);
$registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $control);
app()->instance(RemoteCommandRunner::class, new class($runtime->id) implements RemoteCommandRunner
{
public function __construct(private readonly int $runtimeServerId) {}
public function run(Server $server, string $script): string
{
if ($server->id === $this->runtimeServerId) {
throw new RuntimeException('runtime pull failed');
}
return 'ok';
}
});
$operation = app(CreateManagedRegistrySmokeCheckOperation::class)->execute($registry, $control, [$runtime]);
$runtimeOperation = $operation->children()->firstOrFail();
(new RunStep($runtimeOperation->steps()->firstOrFail()))->handle();
expect($registry->refresh()->health_status)->toBe('unhealthy')
->and($registry->health_message)->toContain('runtime pull failed');
});
it('clears registry auth operation secrets after failed execution', function () {
$organisation = Organisation::factory()->create();
$server = managedRegistryServerFor($organisation);
$registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server);
$registry->markHealthy();
app()->instance(RemoteCommandRunner::class, new class implements RemoteCommandRunner
{
public function run(Server $server, string $script): string
{
throw new RuntimeException('failed');
}
});
$operation = app(CreateRegistryAuthOperation::class)->execute($registry, $server, 'runtime');
$step = $operation->steps()->firstOrFail();
(new RunStep($step))->handle();
expect($step->refresh()->status)->toBe(OperationStatus::FAILED)
->and($step->secrets)->toBeNull();
});
it('marks old successful managed registry artifacts prunable without touching active or retained artifacts', function () {
$organisation = Organisation::factory()->create();
$server = managedRegistryServerFor($organisation);
$registry = app(ManagedRegistryProvisioner::class)->provision(
organisation: $organisation,
url: 'registry.example.com',
controlServer: $server,
retention: 3,
);
$registry->markHealthy();
$application = Application::factory()->for($organisation)->create();
$environment = Environment::factory()->for($application)->create();
Service::factory()->for($environment)->create([
'organisation_id' => $organisation->id,
'available_image_digest' => 'sha256:old-active',
'current_image_digest' => 'sha256:current-active',
]);
foreach (range(1, 6) as $index) {
$environment->buildArtifacts()->create([
'commit_sha' => str_repeat((string) $index, 40),
'image_tag' => "keystone/{$application->uuid}/{$environment->uuid}:{$index}",
'registry_ref' => "registry.example.com/keystone/{$application->uuid}/{$environment->uuid}:{$index}",
'image_digest' => match ($index) {
1 => 'sha256:old-active',
2 => 'sha256:current-active',
default => 'sha256:artifact-'.$index,
},
'status' => BuildArtifactStatus::AVAILABLE,
'created_at' => now()->subMinutes(12 - $index),
'updated_at' => now()->subMinutes(12 - $index),
]);
}
$marked = app(ManagedRegistryRetention::class)->markPrunable($registry);
expect($marked)->toHaveCount(1)
->and($marked->first()->image_digest)->toBe('sha256:artifact-3')
->and($marked->first()->metadata['prune_command'])->toContain('/v2/keystone/')
->and($environment->buildArtifacts()->where('image_digest', 'sha256:old-active')->first()->status)->toBe(BuildArtifactStatus::AVAILABLE)
->and($environment->buildArtifacts()->where('image_digest', 'sha256:current-active')->first()->status)->toBe(BuildArtifactStatus::AVAILABLE);
});
it('blocks managed registry maintenance while matching builds are active', function () {
$organisation = Organisation::factory()->create();
$server = managedRegistryServerFor($organisation);
$registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server);
$application = Application::factory()->for($organisation)->create();
$environment = Environment::factory()->for($application)->create();
$environment->buildArtifacts()->create([
'commit_sha' => str_repeat('a', 40),
'image_tag' => "keystone/{$application->uuid}/{$environment->uuid}:aaaaaaaaaaaa",
'registry_ref' => "registry.example.com/keystone/{$application->uuid}/{$environment->uuid}:aaaaaaaaaaaa",
'status' => BuildArtifactStatus::BUILDING,
]);
expect(fn () => app(CreateManagedRegistryMaintenanceOperation::class)->execute($registry))
->toThrow(RuntimeException::class, 'Managed registry pruning cannot run while builds are active.');
});
it('creates managed registry maintenance operations that delete manifests, run gc, and mark artifacts pruned after execution', function () {
$organisation = Organisation::factory()->create();
$server = managedRegistryServerFor($organisation);
$registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server, retention: 1);
$application = Application::factory()->for($organisation)->create();
$environment = Environment::factory()->for($application)->create();
foreach (range(1, 3) as $index) {
$environment->buildArtifacts()->create([
'commit_sha' => str_repeat((string) $index, 40),
'image_tag' => "keystone/{$application->uuid}/{$environment->uuid}:{$index}",
'registry_ref' => "registry.example.com/keystone/{$application->uuid}/{$environment->uuid}:{$index}",
'image_digest' => 'sha256:artifact-'.$index,
'status' => BuildArtifactStatus::AVAILABLE,
'created_at' => now()->subMinutes(10 - $index),
'updated_at' => now()->subMinutes(10 - $index),
]);
}
app()->instance(RemoteCommandRunner::class, new class implements RemoteCommandRunner
{
public function run(Server $server, string $script): string
{
return 'ok';
}
});
$operation = app(CreateManagedRegistryMaintenanceOperation::class)->execute($registry);
$step = $operation->steps()->firstOrFail();
$unselectedArtifact = $environment->buildArtifacts()->create([
'commit_sha' => str_repeat('9', 40),
'image_tag' => "keystone/{$application->uuid}/{$environment->uuid}:9",
'registry_ref' => "registry.example.com/keystone/{$application->uuid}/{$environment->uuid}:9",
'image_digest' => 'sha256:not-in-this-maintenance-batch',
'status' => BuildArtifactStatus::PRUNABLE,
]);
expect($step->script)->toContain('flock -n 9')
->and($step->script)->toContain('--request DELETE')
->and($step->script)->toContain('delete_failures=0')
->and($step->script)->toContain('status=$(curl')
->and($step->script)->not->toContain('--user "$username:$password"')
->and($step->script)->toContain('Accept: application/vnd.docker.distribution.manifest.v2+json')
->and($step->script)->toContain('garbage-collect --delete-untagged')
->and($step->script)->toContain('docker stop keystone-managed-registry')
->and($step->script)->toContain('docker start keystone-managed-registry')
->and($step->script)->not->toContain($registry->credentials['build_password']);
(new RunStep($step))->handle();
expect($environment->buildArtifacts()->where('status', BuildArtifactStatus::PRUNED)->count())->toBe(2);
expect($unselectedArtifact->refresh()->status)->toBe(BuildArtifactStatus::PRUNABLE);
});
it('keeps managed registry builds blocked until represented smoke checks pass', function () {
$organisation = Organisation::factory()->create();
$server = managedRegistryServerFor($organisation);
$runtime = managedRegistryServerFor($organisation);
$registry = app(ManagedRegistryProvisioner::class)->provision($organisation, 'registry.example.com', $server);
app(CreateManagedRegistrySmokeCheckOperation::class)->execute($registry, $server, [$runtime]);
expect(app(ManagedRegistryHealth::class)->readinessBlocker($registry->refresh()))
->toBe('Managed registry has not passed readiness checks.');
app()->instance(RemoteCommandRunner::class, new class implements RemoteCommandRunner
{
public function run(Server $server, string $script): string
{
return 'ok';
}
});
$operation = $server->operations()->where('kind', OperationKind::REGISTRY_HEALTH_CHECK)->latest()->firstOrFail();
(new RunStep($operation->steps()->firstOrFail()))->handle();
expect($registry->refresh()->health_status)->toBe('healthy')
->and($registry->ready_at)->not->toBeNull()
->and(app(ManagedRegistryHealth::class)->readinessBlocker($registry))->toBeNull();
});
function managedRegistryServerFor(Organisation $organisation): Server
{
$provider = Provider::factory()->forOrganisation($organisation)->create();
$network = Network::create([
'organisation_id' => $organisation->id,
'provider_id' => $provider->id,
'name' => 'test-network',
'ip_range' => '10.0.0.0/24',
]);
return Server::factory()
->forOrganisation($organisation->id)
->forProvider($provider->id)
->forNetwork($network->id)
->create([
'is_control_node' => true,
'build_enabled' => true,
]);
}

View File

@@ -49,6 +49,7 @@ it('shows an operation with steps and children', function () {
'status' => OperationStatus::COMPLETED,
'script' => 'docker ps',
'logs' => 'ok',
'secrets' => ['registry_password_base64' => base64_encode('super-secret-password')],
]);
Operation::factory()->create([
@@ -63,10 +64,13 @@ it('shows an operation with steps and children', function () {
]));
$response->assertOk();
$response->assertDontSee('super-secret-password', false);
$response->assertDontSee('registry_password_base64', false);
$response->assertInertia(fn (AssertableInertia $page) => $page
->component('operations/Show', false)
->where('operation.hash', $operation->hash)
->has('operation.steps', 1)
->missing('operation.steps.0.secrets')
->has('operation.children', 1));
});
@@ -131,6 +135,7 @@ it('cancels operations and downloads logs', function () {
'script' => 'docker ps',
'logs' => 'hello',
'error_logs' => 'error',
'secrets' => ['registry_password_base64' => base64_encode('super-secret-password')],
]);
$this->actingAs($user)->post(route('operations.cancel', [
@@ -142,7 +147,8 @@ it('cancels operations and downloads logs', function () {
]));
expect($operation->refresh()->status)->toBe(OperationStatus::CANCELLED)
->and($operation->finished_at)->not->toBeNull();
->and($operation->finished_at)->not->toBeNull()
->and($operation->steps()->first()->secrets)->toBeNull();
$response = $this->actingAs($user)->get(route('operations.logs', [
'organisation' => $organisation->id,

View File

@@ -19,7 +19,8 @@ it('shows the create registry page', function () {
$response->assertOk();
$response->assertInertia(fn (AssertableInertia $page) => $page
->component('registries/Create', false)
->where('registryTypes.0', RegistryType::GENERIC->value));
->where('registryTypes.0', RegistryType::GENERIC->value)
->where('registryTypes', fn ($types) => ! in_array(RegistryType::MANAGED->value, $types->all(), true)));
});
it('lists registries for an organisation', function () {
@@ -70,6 +71,66 @@ it('stores a registry for multi-server build artifacts', function () {
]);
});
it('rejects user-created managed registries and scheme-prefixed registry urls', function (array $overrides, string $field) {
$user = User::factory()->create();
$organisation = Organisation::factory()->create(['owner_id' => $user->id]);
$this->actingAs($user)->post(route('registries.store', [
'organisation' => $organisation->id,
]), [
'name' => 'Registry',
'type' => RegistryType::GHCR->value,
'url' => 'ghcr.io/example',
'username' => 'keystone',
'password' => 'secret',
...$overrides,
])->assertSessionHasErrors($field);
})->with([
'managed type' => [['type' => RegistryType::MANAGED->value], 'type'],
'url scheme' => [['url' => 'https://registry.example.com'], 'url'],
]);
it('does not delete managed registries through the user registry controller', function () {
$user = User::factory()->create();
$organisation = Organisation::factory()->create(['owner_id' => $user->id]);
$registry = $organisation->registries()->create([
'name' => 'Managed',
'type' => RegistryType::MANAGED,
'url' => 'registry.example.com',
'credentials' => ['username' => 'keystone-build', 'password' => 'secret'],
]);
$this->actingAs($user)
->delete(route('registries.destroy', [$organisation, $registry]))
->assertForbidden();
expect($registry->fresh())->not->toBeNull();
});
it('does not update managed registries through the user registry controller', function () {
$user = User::factory()->create();
$organisation = Organisation::factory()->create(['owner_id' => $user->id]);
$registry = $organisation->registries()->create([
'name' => 'Managed',
'type' => RegistryType::MANAGED,
'url' => 'registry.example.com',
'credentials' => ['username' => 'keystone-build', 'password' => 'secret'],
]);
$this->actingAs($user)
->put(route('registries.update', [$organisation, $registry]), [
'name' => 'GHCR',
'type' => RegistryType::GHCR->value,
'url' => 'ghcr.io/example',
'username' => 'keystone',
'password' => 'secret',
])
->assertForbidden();
expect($registry->refresh()->type)->toBe(RegistryType::MANAGED)
->and($registry->name)->toBe('Managed');
});
it('shows registry usage from published build artifacts', function () {
$user = User::factory()->create();
$organisation = Organisation::factory()->create(['owner_id' => $user->id]);