execute($this->environment); if ($plan->requiresRegistry) { throw new RuntimeException('A registry is required before deploying this environment across multiple servers.'); } if ($plan->blockers !== []) { throw new RuntimeException($plan->blockers[0]); } $operation = $this->environment->operations()->create([ 'kind' => OperationKind::ENVIRONMENT_DEPLOY, 'status' => OperationStatus::PENDING, 'started_at' => now(), ]); $commitSha = $this->targetCommit ?? app(ResolveEnvironmentCommit::class)->execute($this->environment); $services = $this->servicesNeedingDeployment($plan->services, $commitSha); if ($services === []) { $operation->update([ 'status' => OperationStatus::COMPLETED, 'finished_at' => now(), ]); return; } $artifact = app(PlanBuildArtifact::class)->execute($this->environment, $commitSha); $artifact = app(BuildApplicationArtifact::class)->execute($artifact, $operation); foreach ($services as $service) { $service->update([ 'available_image_digest' => $artifact->image_digest, 'desired_revision' => $commitSha, ]); $child = $service->operations()->create([ 'parent_id' => $operation->id, 'kind' => OperationKind::SERVICE_DEPLOY, 'status' => OperationStatus::PENDING, ]); $this->createServiceDeploySteps($child, $service, $commitSha, $artifact->image_digest); $this->createReplicaDeployOperations($child, $service, $artifact->registry_ref); } $this->createGatewayOperations($operation); $this->dispatchChildOperations($operation); } /** * @param array $services * @return array */ private function servicesNeedingDeployment(array $services, string $commitSha): array { return collect($services) ->filter(fn (Service $service): bool => $service->desired_revision !== $commitSha || ! $service->available_image_digest) ->values() ->all(); } private function createServiceDeploySteps(Operation $operation, Service $service, string $commitSha, string $imageDigest): void { foreach ($this->serviceDeployScripts($service, $commitSha, $imageDigest) as $index => $step) { $operation->steps()->create([ 'name' => $step['name'], 'order' => $index + 1, 'status' => OperationStatus::PENDING, 'script' => $step['script'], ]); } } private function createReplicaDeployOperations(Operation $parent, Service $service, ?string $imageReference = null): void { $replicas = $this->ensureServiceReplicas($service); for ($replica = 1; $replica <= max(1, $service->desired_replicas); $replica++) { $serviceReplica = $replicas[$replica - 1] ?? null; $target = $serviceReplica ?: $service; $operation = $target->operations()->create([ 'parent_id' => $parent->id, 'kind' => OperationKind::REPLICA_DEPLOY, 'status' => OperationStatus::PENDING, ]); $serviceReplica?->update([ 'operation_id' => $operation->id, 'image_digest' => $service->available_image_digest, 'status' => 'pending', 'health_status' => 'unknown', ]); foreach ($this->replicaDeployScripts($service, $replica, $imageReference) as $index => $step) { $operation->steps()->create([ 'name' => $step['name'], 'order' => $index + 1, 'status' => OperationStatus::PENDING, 'script' => $step['script'], ]); } } } /** * @return array */ private function ensureServiceReplicas(Service $service): array { $service->loadMissing('replicas'); $serverIds = $this->placementServerIds($service); if ($service->replicas->count() < $service->desired_replicas && $serverIds !== []) { for ($index = $service->replicas->count() + 1; $index <= $service->desired_replicas; $index++) { $service->replicas()->create([ 'server_id' => $serverIds[($index - 1) % count($serverIds)], 'container_name' => "keystone-service-{$service->id}-{$index}", 'internal_host' => "keystone-service-{$service->id}", 'internal_port' => $this->defaultInternalPort($service), 'status' => 'pending', 'health_status' => 'unknown', 'config' => [], ]); } $service->load('replicas'); } return $service->replicas ->take(max(1, $service->desired_replicas)) ->values() ->all(); } /** * @return array */ private function placementServerIds(Service $service): array { $configured = collect($service->config['server_ids'] ?? []) ->map(fn (mixed $serverId): int => (int) $serverId) ->filter() ->unique() ->values() ->all(); if ($configured !== []) { return $configured; } $existing = $service->replicas ->pluck('server_id') ->filter() ->unique() ->values() ->all(); if ($existing !== []) { return $existing; } return $service->server_id ? [(int) $service->server_id] : []; } private function createGatewayOperations(Operation $parent): void { $this->environment->loadMissing('attachments.service.replicas', 'attachments.serviceSlice'); foreach ($this->environment->attachments->where('role', EnvironmentAttachmentRole::GATEWAY) as $attachment) { $target = $attachment->serviceSlice ?: $this->environment; $sliceConfigure = $target->operations()->create([ 'parent_id' => $parent->id, 'kind' => OperationKind::SLICE_CONFIGURE, 'status' => OperationStatus::PENDING, ]); $sliceConfigure->steps()->create([ 'name' => 'Configure Caddy route', 'order' => 1, 'status' => OperationStatus::PENDING, 'script' => $this->configureCaddyRouteScript($attachment), ]); $gatewayCutover = $this->environment->operations()->create([ 'parent_id' => $parent->id, 'kind' => OperationKind::GATEWAY_CUTOVER, 'status' => OperationStatus::PENDING, ]); foreach ($this->gatewayCutoverSteps($attachment) as $index => $step) { $gatewayCutover->steps()->create([ 'name' => $step['name'], 'order' => $index + 1, 'status' => OperationStatus::PENDING, 'script' => $step['script'], ]); } } } /** * @return array */ private function serviceDeployScripts(Service $service, string $commitSha, string $imageDigest): array { $servicePath = $this->servicePath($service); $composePath = "{$servicePath}/compose.yml"; $serviceKey = $this->serviceKey($service); $steps = [ [ 'name' => 'Resolve target commit', 'script' => implode("\n", [ "mkdir -p {$servicePath}", 'printf %s '.escapeshellarg($commitSha)." > {$servicePath}/REVISION", ]), ], [ 'name' => 'Create or reuse build artifact', 'script' => 'printf %s '.escapeshellarg($imageDigest)." > {$servicePath}/IMAGE_DIGEST", ], [ 'name' => 'Render Compose files', 'script' => $this->composeUploadScript($service), ], ]; foreach ($service->driver()->preSwitchSteps() as $step) { $steps[] = [ 'name' => $step->name, 'script' => $step->getScriptTemplate(), ]; } if (($service->config['migration_timing'] ?? 'pre_switch') === 'pre_switch') { $steps[] = [ 'name' => 'Run migrations', 'script' => app(BuildMigrationScript::class)->execute($service), ]; } $steps = [ ...$steps, [ 'name' => 'Deploy replicas', 'script' => "docker compose -f {$composePath} up -d --scale {$serviceKey}=".max(1, $service->desired_replicas), ], [ 'name' => 'Health check replicas', 'script' => "docker compose -f {$composePath} ps --status running {$serviceKey}", ], ]; if (($service->config['migration_timing'] ?? 'pre_switch') === 'post_switch') { $steps[] = [ 'name' => 'Run migrations', 'script' => app(BuildMigrationScript::class)->execute($service), ]; } return [ ...$steps, [ 'name' => 'Drain old replicas', 'script' => "docker ps --filter 'label=keystone.service_id={$service->id}' --filter 'label=keystone.draining=true' --format '{{.ID}}' | xargs -r docker stop", ], ]; } /** * @return array */ private function replicaDeployScripts(Service $service, int $replica, ?string $imageReference = null): array { $composePath = $this->servicePath($service).'/compose.yml'; $project = "keystone_service_{$service->id}_replica_{$replica}"; $serviceKey = $this->serviceKey($service); $steps = []; if ($imageReference && $service->available_image_digest) { $steps[] = [ 'name' => "Pull image for replica {$replica}", 'script' => 'docker pull '.escapeshellarg($imageReference.'@'.$service->available_image_digest), ]; } return [ ...$steps, [ 'name' => "Render replica {$replica}", 'script' => "docker compose -p {$project} -f {$composePath} config --quiet", ], [ 'name' => "Start replica {$replica}", 'script' => implode("\n", [ "docker compose -p {$project} -f {$composePath} up -d {$serviceKey}", "container_id=$(docker compose -p {$project} -f {$composePath} ps -q {$serviceKey})", 'printf "container_id=%s\n" "$container_id"', ]), ], [ 'name' => "Health check replica {$replica}", 'script' => implode("\n", [ "docker compose -p {$project} -f {$composePath} ps --status running {$serviceKey}", "container_id=$(docker compose -p {$project} -f {$composePath} ps -q {$serviceKey})", 'health_status=$(docker inspect --format "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}" "$container_id")', 'printf "health_status=%s\n" "$health_status"', ]), ], ]; } private function composeUploadScript(Service $service): string { $servicePath = $this->servicePath($service); try { $renderer = app(ComposeRenderer::class); $compose = $renderer->render($service); $env = $renderer->renderEnvironmentFile($service); } catch (InvalidArgumentException) { $compose = "services:\n {$this->serviceKey($service)}:\n image: \"{$service->available_image_digest}\"\n"; $env = ''; } return implode("\n", [ "mkdir -p {$servicePath}", 'printf %s '.escapeshellarg(base64_encode($compose))." | base64 -d > {$servicePath}/compose.yml", 'printf %s '.escapeshellarg(base64_encode($env))." | base64 -d > {$servicePath}/.env", ]); } /** * @return array */ private function gatewayCutoverSteps(EnvironmentAttachment $attachment): array { $containerName = $attachment->service->replicas()->first()?->container_name; $config = $attachment->serviceSlice?->config ?? []; $domain = $config['domain'] ?? null; $tlsEnabled = $config['tls_enabled'] ?? true; $reloadCommand = $containerName ? 'docker exec '.escapeshellarg($containerName).' caddy reload --config /etc/caddy/Caddyfile' : "docker compose -f /home/keystone/services/{$attachment->service_id}/compose.yml exec -T {$this->serviceKey($attachment->service)} caddy reload --config /etc/caddy/Caddyfile"; $certificateCheck = $tlsEnabled && $domain ? 'curl --fail --silent --show-error --head https://'.escapeshellarg($domain).' >/dev/null' : 'true # TLS disabled or no domain configured for this route'; return [ [ 'name' => 'Validate Caddy route configuration', 'script' => 'test -s /home/keystone/gateway/Caddyfile', ], [ 'name' => 'Check TLS certificate status', 'script' => $certificateCheck, ], [ 'name' => 'Reload Caddy', 'script' => $reloadCommand, ], [ 'name' => 'Verify new upstreams are reachable', 'script' => 'curl --fail --silent --show-error http://127.0.0.1/ >/dev/null || true', ], [ 'name' => 'Drain old upstreams', 'script' => implode("\n", [ "docker ps --filter 'label=keystone.environment_id={$this->environment->id}' --filter 'label=keystone.draining=true' --format '{{.ID}}' | xargs -r docker stop --time 30", ]), ], ]; } private function configureCaddyRouteScript(EnvironmentAttachment $attachment): string { $upstreams = $this->gatewayUpstreams($attachment); $caddyfile = app(CaddyRouteRenderer::class)->render($attachment, $upstreams); return implode("\n", [ 'mkdir -p /home/keystone/gateway/Caddyfile.d', "cat > /home/keystone/gateway/Caddyfile.d/{$attachment->id}.caddy <<'KEYSTONE_CADDY_ROUTE'", $caddyfile, 'KEYSTONE_CADDY_ROUTE', 'cat /home/keystone/gateway/Caddyfile.d/*.caddy > /home/keystone/gateway/Caddyfile', ]); } /** * @return array */ private function gatewayUpstreams(EnvironmentAttachment $attachment): array { $gatewayReplica = $attachment->service->replicas()->first(); return $this->environment->services() ->where('type', \App\Enums\ServiceType::LARAVEL) ->get() ->filter(fn (Service $service): bool => in_array('web', $service->process_roles ?? [], true)) ->flatMap(function (Service $service) use ($gatewayReplica) { return $service->replicas ->map(function (ServiceReplica $replica) use ($gatewayReplica) { $endpoint = app(RegisterServiceEndpoint::class)->execute( replica: $replica, consumerReplica: $gatewayReplica, allowPublicFallback: false, ); return [ 'priority' => $endpoint->priority, 'target' => $this->endpointTarget($endpoint->scope, $endpoint->hostname, $endpoint->port), ]; }); }) ->sortBy('priority') ->pluck('target') ->values() ->whenEmpty(fn ($targets) => $targets->push('web:80')) ->all(); } private function endpointTarget(ServiceEndpointScope $scope, string $hostname, int $port): string { return $scope === ServiceEndpointScope::DOCKER_NETWORK ? $hostname.':'.$port : 'http://'.$hostname.':'.$port; } private function dispatchChildOperations(Operation $operation): void { $operation->update(['status' => OperationStatus::IN_PROGRESS]); $operation->children() ->with('steps') ->orderBy('id') ->get() ->first(fn (Operation $child): bool => $child->steps->isNotEmpty()) ?->steps ->sortBy('order') ->first() ?->dispatchJob(); } private function servicePath(Service $service): string { return "/home/keystone/services/{$service->id}"; } private function serviceKey(Service $service): string { return str($service->name)->slug('_')->value() ?: 'service'; } private function defaultInternalPort(Service $service): int { return match ($service->type) { \App\Enums\ServiceType::POSTGRES => 5432, \App\Enums\ServiceType::VALKEY => 6379, \App\Enums\ServiceType::CADDY, \App\Enums\ServiceType::LARAVEL => 80, }; } }