instance(RemoteCommandRunner::class, new class implements RemoteCommandRunner { public function run(Server $server, string $script): string { return "image_digest=billing-api:aaaaaaaaaaaa@sha256:deploymentdigest\n"; } }); Process::fake(function ($process) { $command = is_array($process->command) ? implode(' ', $process->command) : $process->command; return match (true) { str_contains($command, 'git ls-remote') => Process::result(output: str_repeat('a', 40)."\trefs/heads/main\n"), str_contains($command, 'docker image inspect') => Process::result(output: "billing-api:aaaaaaaaaaaa@sha256:deploymentdigest\n"), default => Process::result(), }; }); }); it('creates a parent environment operation with child service deploy operations', function () { $organisation = Organisation::factory()->create(); $provider = Provider::factory()->forOrganisation($organisation)->create(); $network = Network::create([ 'organisation_id' => $organisation->id, 'provider_id' => $provider->id, 'name' => 'test-network', 'ip_range' => '10.0.0.0/24', ]); $server = Server::factory() ->forOrganisation($organisation->id) ->forProvider($provider->id) ->forNetwork($network->id) ->create(); $application = Application::factory()->for($organisation)->create(['name' => 'Keystone Test App']); generateDeployKey($application); $environment = Environment::factory()->for($application)->create(); $service = Service::factory()->for($environment)->for($server)->create([ 'organisation_id' => $organisation->id, 'name' => 'web', 'category' => ServiceCategory::APPLICATION, 'type' => ServiceType::LARAVEL, 'version' => 'php-8.4', 'version_track' => 'php-8.4', 'driver_name' => 'laravel.php-8.4', 'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT, ]); (new DeployEnvironment($environment))->handle(); $parent = Operation::query() ->where('target_type', $environment->getMorphClass()) ->where('target_id', $environment->id) ->first(); $child = Operation::query() ->where('parent_id', $parent->id) ->where('target_type', $service->getMorphClass()) ->where('target_id', $service->id) ->first(); expect($parent->kind)->toBe(OperationKind::ENVIRONMENT_DEPLOY) ->and($child->kind)->toBe(OperationKind::SERVICE_DEPLOY) ->and($child->steps)->toHaveCount(7) ->and($child->steps()->where('name', 'Render Compose files')->first()->script)->toContain("base64 -d > /home/keystone/services/{$service->id}/compose.yml") ->and($child->steps()->where('name', 'Render Compose files')->first()->script)->toContain("base64 -d > /home/keystone/services/{$service->id}/.env") ->and($child->steps()->where('name', 'Run migrations')->first()->script)->toContain('docker compose -f /home/keystone/services/'.$service->id.'/compose.yml run --rm web php artisan migrate --force') ->and($child->steps()->where('name', 'Deploy replicas')->first()->script)->toContain('docker compose -f /home/keystone/services/'.$service->id.'/compose.yml up -d --scale web=1') ->and($child->steps()->where('name', 'Update gateway routes')->exists())->toBeFalse() ->and($child->steps()->pluck('script')->implode("\n"))->not->toContain('echo ') ->and($environment->buildArtifacts()->first()->image_digest)->toBe('sha256:deploymentdigest') ->and($service->refresh()->available_image_digest)->toBe('sha256:deploymentdigest') ->and($service->desired_revision)->toBe(str_repeat('a', 40)); Bus::assertDispatched(RunStep::class); }); it('creates replica route configure and gateway cutover child operations', function () { $organisation = Organisation::factory()->create(); $provider = Provider::factory()->forOrganisation($organisation)->create(); $network = Network::create([ 'organisation_id' => $organisation->id, 'provider_id' => $provider->id, 'name' => 'test-network', 'ip_range' => '10.0.0.0/24', ]); $server = Server::factory() ->forOrganisation($organisation->id) ->forProvider($provider->id) ->forNetwork($network->id) ->create(); $application = Application::factory()->for($organisation)->create(['name' => 'Keystone Test App']); generateDeployKey($application); $environment = Environment::factory()->for($application)->create(); $web = Service::factory()->for($environment)->for($server)->create([ 'organisation_id' => $organisation->id, 'name' => 'web', 'category' => ServiceCategory::APPLICATION, 'type' => ServiceType::LARAVEL, 'version' => 'php-8.4', 'version_track' => 'php-8.4', 'driver_name' => 'laravel.php-8.4', 'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT, 'process_roles' => ['web'], 'desired_replicas' => 2, ]); $web->replicas()->create([ 'server_id' => $server->id, 'container_name' => 'web-1', 'internal_host' => 'web-1', 'internal_port' => 80, 'status' => 'running', 'health_status' => 'healthy', 'config' => [], ]); $web->replicas()->create([ 'server_id' => $server->id, 'container_name' => 'web-2', 'internal_host' => 'web-2', 'internal_port' => 80, 'status' => 'running', 'health_status' => 'healthy', 'config' => [], ]); $gateway = Service::factory()->for($environment)->for($server)->create([ 'organisation_id' => $organisation->id, 'name' => 'gateway', 'category' => ServiceCategory::GATEWAY, 'type' => ServiceType::CADDY, 'version' => '2', 'version_track' => '2', 'driver_name' => 'caddy.2', 'deploy_policy' => DeployPolicy::MANUAL_OR_ON_ROUTE_CHANGE, ]); $gateway->replicas()->create([ 'server_id' => $server->id, 'container_name' => 'gateway-1', 'internal_host' => 'gateway-1', 'internal_port' => 80, 'status' => 'running', 'health_status' => 'healthy', 'config' => [], ]); $route = ServiceSlice::factory()->for($gateway)->create([ 'environment_id' => $environment->id, 'type' => 'route', 'name' => 'example.com', ]); $environment->attachments()->create([ 'service_id' => $gateway->id, 'service_slice_id' => $route->id, 'role' => EnvironmentAttachmentRole::GATEWAY, 'is_primary' => true, ]); $organisation->registries()->create([ 'name' => 'registry', 'type' => 'generic', 'url' => 'registry.example.com', ]); (new DeployEnvironment($environment))->handle(); $parent = Operation::query() ->where('target_type', $environment->getMorphClass()) ->where('target_id', $environment->id) ->where('kind', OperationKind::ENVIRONMENT_DEPLOY) ->first(); $serviceDeploy = $parent->children()->where('kind', OperationKind::SERVICE_DEPLOY)->first(); $compose = renderedComposeFrom($serviceDeploy->steps()->where('name', 'Render Compose files')->first()->script, $web->id); expect($serviceDeploy->children()->where('kind', OperationKind::REPLICA_DEPLOY)->count())->toBe(2) ->and($parent->children()->where('kind', OperationKind::SLICE_CONFIGURE)->count())->toBe(1) ->and($parent->children()->where('kind', OperationKind::GATEWAY_CUTOVER)->count())->toBe(1) ->and($serviceDeploy->children()->where('kind', OperationKind::REPLICA_DEPLOY)->first()->steps()->where('name', 'Start replica 1')->first()->script) ->toContain('docker compose -p keystone_service_') ->and($serviceDeploy->children()->where('kind', OperationKind::REPLICA_DEPLOY)->first()->steps()->where('name', 'Start replica 1')->first()->script) ->toContain('container_id=') ->and($serviceDeploy->children()->where('kind', OperationKind::REPLICA_DEPLOY)->first()->steps()->where('name', 'Pull image for replica 1')->first()->script) ->toContain('docker pull') ->and($serviceDeploy->children()->where('kind', OperationKind::REPLICA_DEPLOY)->first()->steps()->where('name', 'Pull image for replica 1')->first()->script) ->toContain('@sha256:deploymentdigest') ->and($compose) ->toContain('image: "registry.example.com/keystone-test-app:aaaaaaaaaaaa@sha256:deploymentdigest"') ->and($serviceDeploy->children()->where('kind', OperationKind::REPLICA_DEPLOY)->first()->steps()->where('name', 'Health check replica 1')->first()->script) ->toContain('health_status=') ->and($parent->children()->where('kind', OperationKind::SLICE_CONFIGURE)->first()->steps()->first()->script) ->toContain('/home/keystone/gateway/Caddyfile.d') ->and($parent->children()->where('kind', OperationKind::SLICE_CONFIGURE)->first()->steps()->first()->script) ->toContain('reverse_proxy web-1:80 web-2:80') ->and($web->endpoints()->count())->toBe(2) ->and($parent->children()->where('kind', OperationKind::GATEWAY_CUTOVER)->first()->steps()->pluck('name')->all()) ->toBe([ 'Validate Caddy route configuration', 'Check TLS certificate status', 'Reload Caddy', 'Verify new upstreams are reachable', 'Drain old upstreams', ]) ->and($parent->children()->where('kind', OperationKind::GATEWAY_CUTOVER)->first()->steps()->where('name', 'Reload Caddy')->first()->script) ->toContain('gateway-1'); }); it('honors manual disabled and post-switch migration settings', function (string $mode, string $timing, string $expectedScript, int $expectedOrder) { $organisation = Organisation::factory()->create(); $provider = Provider::factory()->forOrganisation($organisation)->create(); $network = Network::create([ 'organisation_id' => $organisation->id, 'provider_id' => $provider->id, 'name' => 'test-network', 'ip_range' => '10.0.0.0/24', ]); $server = Server::factory() ->forOrganisation($organisation->id) ->forProvider($provider->id) ->forNetwork($network->id) ->create(); $application = Application::factory()->for($organisation)->create(); generateDeployKey($application); $environment = Environment::factory()->for($application)->create(); $service = Service::factory()->for($environment)->for($server)->create([ 'organisation_id' => $organisation->id, 'name' => 'web', 'category' => ServiceCategory::APPLICATION, 'type' => ServiceType::LARAVEL, 'version' => 'php-8.4', 'version_track' => 'php-8.4', 'driver_name' => 'laravel.php-8.4', 'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT, 'config' => [ 'migration_mode' => $mode, 'migration_timing' => $timing, 'migration_command' => 'php artisan migrate --force', ], ]); (new DeployEnvironment($environment))->handle(); $step = $service->operations()->where('kind', OperationKind::SERVICE_DEPLOY)->first()->steps()->where('name', 'Run migrations')->first(); $expectedScript = $expectedScript === 'migration' ? "docker compose -f /home/keystone/services/{$service->id}/compose.yml run --rm web php artisan migrate --force" : $expectedScript; expect($step->script)->toBe($expectedScript) ->and($step->order)->toBe($expectedOrder); })->with([ 'manual pre-switch' => ['manual', 'pre_switch', 'true', 4], 'disabled pre-switch' => ['disabled', 'pre_switch', 'true', 4], 'auto post-switch' => ['auto', 'post_switch', 'migration', 6], ]); it('assigns replica operations and artifact metadata to service replicas', function () { $organisation = Organisation::factory()->create(); $provider = Provider::factory()->forOrganisation($organisation)->create(); $network = Network::create([ 'organisation_id' => $organisation->id, 'provider_id' => $provider->id, 'name' => 'test-network', 'ip_range' => '10.0.0.0/24', ]); $server = Server::factory() ->forOrganisation($organisation->id) ->forProvider($provider->id) ->forNetwork($network->id) ->create(); $application = Application::factory()->for($organisation)->create(); generateDeployKey($application); $environment = Environment::factory()->for($application)->create(); $service = Service::factory()->for($environment)->for($server)->create([ 'organisation_id' => $organisation->id, 'name' => 'web', 'category' => ServiceCategory::APPLICATION, 'type' => ServiceType::LARAVEL, 'version' => 'php-8.4', 'version_track' => 'php-8.4', 'driver_name' => 'laravel.php-8.4', 'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT, 'desired_replicas' => 2, ]); $organisation->registries()->create([ 'name' => 'registry', 'type' => 'generic', 'url' => 'registry.example.com', ]); (new DeployEnvironment($environment))->handle(); $replicas = $service->replicas()->orderBy('id')->get(); expect($replicas)->toHaveCount(2) ->and($replicas[0]->operation_id)->not->toBeNull() ->and($replicas[0]->image_digest)->toBe('sha256:deploymentdigest') ->and($replicas[0]->status)->toBe('pending') ->and($replicas[0]->health_status)->toBe('unknown') ->and($replicas[0]->operation->kind)->toBe(OperationKind::REPLICA_DEPLOY) ->and($replicas[0]->operation->parent->kind)->toBe(OperationKind::SERVICE_DEPLOY) ->and($replicas[0]->operation->target->is($replicas[0]))->toBeTrue(); }); it('places desired replicas across configured server placements', function () { $organisation = Organisation::factory()->create(); $provider = Provider::factory()->forOrganisation($organisation)->create(); $network = Network::create([ 'organisation_id' => $organisation->id, 'provider_id' => $provider->id, 'name' => 'test-network', 'ip_range' => '10.0.0.0/24', ]); $servers = Server::factory() ->count(2) ->forOrganisation($organisation->id) ->forProvider($provider->id) ->forNetwork($network->id) ->create(); $application = Application::factory()->for($organisation)->create(); generateDeployKey($application); $environment = Environment::factory()->for($application)->create(); $service = Service::factory()->for($environment)->create([ 'organisation_id' => $organisation->id, 'name' => 'web', 'category' => ServiceCategory::APPLICATION, 'type' => ServiceType::LARAVEL, 'version' => 'php-8.4', 'version_track' => 'php-8.4', 'driver_name' => 'laravel.php-8.4', 'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT, 'desired_replicas' => 2, 'config' => [ 'server_ids' => $servers->pluck('id')->all(), ], ]); $organisation->registries()->create([ 'name' => 'registry', 'type' => 'generic', 'url' => 'registry.example.com', ]); (new DeployEnvironment($environment))->handle(); expect($service->replicas()->pluck('server_id')->all()) ->toBe($servers->pluck('id')->all()); }); it('renders compose and root registry auth on each managed registry replica server', function () { $organisation = Organisation::factory()->create(); $provider = Provider::factory()->forOrganisation($organisation)->create(); $network = Network::create([ 'organisation_id' => $organisation->id, 'provider_id' => $provider->id, 'name' => 'test-network', 'ip_range' => '10.0.0.0/24', ]); $servers = Server::factory() ->count(2) ->forOrganisation($organisation->id) ->forProvider($provider->id) ->forNetwork($network->id) ->create(); $servers[0]->update([ 'is_control_node' => true, 'build_enabled' => true, ]); $organisation->registries()->create([ 'name' => 'Managed', 'type' => 'managed', 'url' => 'registry.example.com', 'credentials' => [ 'build_username' => 'keystone-build', 'build_password' => 'build-secret', 'runtime_username' => 'keystone-runtime', 'runtime_password' => 'runtime-secret', ], 'control_server_id' => $servers[0]->id, 'health_status' => 'healthy', 'readiness_checks' => ['control_https' => 'passed', 'build_push' => 'passed'], 'ready_at' => now(), ]); $application = Application::factory()->for($organisation)->create(['name' => 'Keystone Test App']); generateDeployKey($application); $environment = Environment::factory()->for($application)->create(); $service = Service::factory()->for($environment)->create([ 'organisation_id' => $organisation->id, 'name' => 'web', 'category' => ServiceCategory::APPLICATION, 'type' => ServiceType::LARAVEL, 'version' => 'php-8.4', 'version_track' => 'php-8.4', 'driver_name' => 'laravel.php-8.4', 'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT, 'desired_replicas' => 2, 'config' => [ 'server_ids' => $servers->pluck('id')->all(), ], ]); (new DeployEnvironment($environment))->handle(); $serviceDeploy = $service->operations() ->where('kind', OperationKind::SERVICE_DEPLOY) ->firstOrFail(); $replicaOperations = $serviceDeploy->children() ->where('kind', OperationKind::REPLICA_DEPLOY) ->with('steps') ->get(); expect($replicaOperations)->toHaveCount(2) ->and($replicaOperations[0]->steps->pluck('name')->all())->toContain('Render replica 1 Compose files') ->and($replicaOperations[1]->steps->pluck('name')->all())->toContain('Render replica 2 Compose files'); $authStep = $replicaOperations[0]->steps->firstWhere('name', 'Configure registry auth for replica 1'); expect($authStep->script)->toContain("DOCKER_CONFIG='/root/.docker'") ->and($authStep->script)->toContain('[!registry_password_base64!]') ->and($authStep->script)->not->toContain('runtime-secret') ->and($authStep->secrets['registry_password_base64'])->toBe(base64_encode('runtime-secret')); }); it('skips environment service operations when the target revision is already available', function () { $organisation = Organisation::factory()->create(); $application = Application::factory()->for($organisation)->create(); generateDeployKey($application); $environment = Environment::factory()->for($application)->create(); Service::factory()->for($environment)->create([ 'organisation_id' => $organisation->id, 'name' => 'web', 'category' => ServiceCategory::APPLICATION, 'type' => ServiceType::LARAVEL, 'version' => 'php-8.4', 'version_track' => 'php-8.4', 'driver_name' => 'laravel.php-8.4', 'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT, 'desired_revision' => str_repeat('a', 40), 'available_image_digest' => 'sha256:existing', ]); (new DeployEnvironment($environment))->handle(); $parent = Operation::query() ->where('target_type', $environment->getMorphClass()) ->where('target_id', $environment->id) ->where('kind', OperationKind::ENVIRONMENT_DEPLOY) ->first(); expect($parent->status)->toBe(\App\Enums\OperationStatus::COMPLETED) ->and($parent->children()->count())->toBe(0) ->and($environment->buildArtifacts()->count())->toBe(0); }); function generateDeployKey(Application $application): void { app(GenerateDeployKey::class)->execute($application, [ 'public' => 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITestPublicKey keystone', 'private' => "-----BEGIN OPENSSH PRIVATE KEY-----\ntest\n-----END OPENSSH PRIVATE KEY-----", 'fingerprint' => 'SHA256:test', ]); } function renderedComposeFrom(string $script, int $serviceId): string { preg_match( '/printf %s \'(?[^\']+)\' \| base64 -d > \/home\/keystone\/services\/'.$serviceId.'\/compose\.yml/', $script, $matches, ); return base64_decode($matches['encoded'] ?? '', true) ?: ''; } it('blocks multi-server deploys that do not have a registry', function () { $organisation = Organisation::factory()->create(); $application = Application::factory()->for($organisation)->create(); $environment = Environment::factory()->for($application)->create(); Service::factory()->for($environment)->create([ 'organisation_id' => $organisation->id, 'category' => ServiceCategory::APPLICATION, 'type' => ServiceType::LARAVEL, 'version' => 'php-8.4', 'version_track' => 'php-8.4', 'driver_name' => 'laravel.php-8.4', 'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT, 'desired_replicas' => 2, ]); expect(fn () => (new DeployEnvironment($environment))->handle()) ->toThrow(RuntimeException::class, 'A registry is required before deploying this environment across multiple servers.'); }); it('blocks deployment when single scheduler mode would run on multiple replicas', function () { $organisation = Organisation::factory()->create(); $application = Application::factory()->for($organisation)->create(); generateDeployKey($application); $environment = Environment::factory()->for($application)->create([ 'scheduler_enabled' => true, 'scheduler_mode' => \App\Enums\SchedulerMode::SINGLE, ]); $web = Service::factory()->for($environment)->create([ 'organisation_id' => $organisation->id, 'name' => 'web', 'category' => ServiceCategory::APPLICATION, 'type' => ServiceType::LARAVEL, 'version' => 'php-8.4', 'version_track' => 'php-8.4', 'driver_name' => 'laravel.php-8.4', 'deploy_policy' => DeployPolicy::WITH_ENVIRONMENT, 'process_roles' => ['web', 'scheduler'], 'desired_replicas' => 2, ]); $environment->forceFill(['scheduler_target_service_id' => $web->id])->save(); $organisation->registries()->create([ 'name' => 'registry', 'type' => 'generic', 'url' => 'registry.example.com', ]); expect(fn () => (new DeployEnvironment($environment->refresh()))->handle()) ->toThrow(RuntimeException::class, 'Scheduler mode single requires the scheduler target service to run exactly one replica.'); });