Implement Keystone environment deployments
This commit is contained in:
497
app/Jobs/Environments/DeployEnvironment.php
Normal file
497
app/Jobs/Environments/DeployEnvironment.php
Normal file
@@ -0,0 +1,497 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs\Environments;
|
||||
|
||||
use App\Actions\Environments\BuildApplicationArtifact;
|
||||
use App\Actions\Environments\BuildMigrationScript;
|
||||
use App\Actions\Environments\PlanBuildArtifact;
|
||||
use App\Actions\Environments\PlanEnvironmentDeployment;
|
||||
use App\Actions\Environments\ResolveEnvironmentCommit;
|
||||
use App\Actions\Services\RegisterServiceEndpoint;
|
||||
use App\Enums\EnvironmentAttachmentRole;
|
||||
use App\Enums\OperationKind;
|
||||
use App\Enums\OperationStatus;
|
||||
use App\Enums\ServiceEndpointScope;
|
||||
use App\Models\Environment;
|
||||
use App\Models\EnvironmentAttachment;
|
||||
use App\Models\Operation;
|
||||
use App\Models\Service;
|
||||
use App\Models\ServiceReplica;
|
||||
use App\Services\Compose\ComposeRenderer;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Queue\Queueable;
|
||||
use InvalidArgumentException;
|
||||
use RuntimeException;
|
||||
|
||||
class DeployEnvironment implements ShouldQueue
|
||||
{
|
||||
use Queueable;
|
||||
|
||||
public function __construct(
|
||||
public Environment $environment,
|
||||
) {
|
||||
//
|
||||
}
|
||||
|
||||
public function handle(): void
|
||||
{
|
||||
$plan = app(PlanEnvironmentDeployment::class)->execute($this->environment);
|
||||
|
||||
if ($plan->requiresRegistry) {
|
||||
throw new RuntimeException('A registry is required before deploying this environment across multiple servers.');
|
||||
}
|
||||
|
||||
if ($plan->blockers !== []) {
|
||||
throw new RuntimeException($plan->blockers[0]);
|
||||
}
|
||||
|
||||
$operation = $this->environment->operations()->create([
|
||||
'kind' => OperationKind::ENVIRONMENT_DEPLOY,
|
||||
'status' => OperationStatus::PENDING,
|
||||
'started_at' => now(),
|
||||
]);
|
||||
|
||||
$commitSha = app(ResolveEnvironmentCommit::class)->execute($this->environment);
|
||||
$services = $this->servicesNeedingDeployment($plan->services, $commitSha);
|
||||
|
||||
if ($services === []) {
|
||||
$operation->update([
|
||||
'status' => OperationStatus::COMPLETED,
|
||||
'finished_at' => now(),
|
||||
]);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$artifact = app(PlanBuildArtifact::class)->execute($this->environment, $commitSha);
|
||||
$artifact = app(BuildApplicationArtifact::class)->execute($artifact, $operation);
|
||||
|
||||
foreach ($services as $service) {
|
||||
$service->update([
|
||||
'available_image_digest' => $artifact->image_digest,
|
||||
'desired_revision' => $commitSha,
|
||||
]);
|
||||
|
||||
$child = $service->operations()->create([
|
||||
'parent_id' => $operation->id,
|
||||
'kind' => OperationKind::SERVICE_DEPLOY,
|
||||
'status' => OperationStatus::PENDING,
|
||||
]);
|
||||
|
||||
$this->createServiceDeploySteps($child, $service, $commitSha, $artifact->image_digest);
|
||||
$this->createReplicaDeployOperations($child, $service, $artifact->registry_ref);
|
||||
}
|
||||
|
||||
$this->createGatewayOperations($operation);
|
||||
$this->dispatchChildOperations($operation);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, Service> $services
|
||||
* @return array<int, Service>
|
||||
*/
|
||||
private function servicesNeedingDeployment(array $services, string $commitSha): array
|
||||
{
|
||||
return collect($services)
|
||||
->filter(fn (Service $service): bool => $service->desired_revision !== $commitSha || ! $service->available_image_digest)
|
||||
->values()
|
||||
->all();
|
||||
}
|
||||
|
||||
private function createServiceDeploySteps(Operation $operation, Service $service, string $commitSha, string $imageDigest): void
|
||||
{
|
||||
foreach ($this->serviceDeployScripts($service, $commitSha, $imageDigest) as $index => $step) {
|
||||
$operation->steps()->create([
|
||||
'name' => $step['name'],
|
||||
'order' => $index + 1,
|
||||
'status' => OperationStatus::PENDING,
|
||||
'script' => $step['script'],
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
private function createReplicaDeployOperations(Operation $parent, Service $service, ?string $imageReference = null): void
|
||||
{
|
||||
$replicas = $this->ensureServiceReplicas($service);
|
||||
|
||||
for ($replica = 1; $replica <= max(1, $service->desired_replicas); $replica++) {
|
||||
$serviceReplica = $replicas[$replica - 1] ?? null;
|
||||
$target = $serviceReplica ?: $service;
|
||||
|
||||
$operation = $target->operations()->create([
|
||||
'parent_id' => $parent->id,
|
||||
'kind' => OperationKind::REPLICA_DEPLOY,
|
||||
'status' => OperationStatus::PENDING,
|
||||
]);
|
||||
|
||||
$serviceReplica?->update([
|
||||
'operation_id' => $operation->id,
|
||||
'image_digest' => $service->available_image_digest,
|
||||
'status' => 'pending',
|
||||
'health_status' => 'unknown',
|
||||
]);
|
||||
|
||||
foreach ($this->replicaDeployScripts($service, $replica, $imageReference) as $index => $step) {
|
||||
$operation->steps()->create([
|
||||
'name' => $step['name'],
|
||||
'order' => $index + 1,
|
||||
'status' => OperationStatus::PENDING,
|
||||
'script' => $step['script'],
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, ServiceReplica>
|
||||
*/
|
||||
private function ensureServiceReplicas(Service $service): array
|
||||
{
|
||||
$service->loadMissing('replicas');
|
||||
|
||||
$serverIds = $this->placementServerIds($service);
|
||||
|
||||
if ($service->replicas->count() < $service->desired_replicas && $serverIds !== []) {
|
||||
for ($index = $service->replicas->count() + 1; $index <= $service->desired_replicas; $index++) {
|
||||
$service->replicas()->create([
|
||||
'server_id' => $serverIds[($index - 1) % count($serverIds)],
|
||||
'container_name' => "keystone-service-{$service->id}-{$index}",
|
||||
'internal_host' => "keystone-service-{$service->id}",
|
||||
'internal_port' => $this->defaultInternalPort($service),
|
||||
'status' => 'pending',
|
||||
'health_status' => 'unknown',
|
||||
'config' => [],
|
||||
]);
|
||||
}
|
||||
|
||||
$service->load('replicas');
|
||||
}
|
||||
|
||||
return $service->replicas
|
||||
->take(max(1, $service->desired_replicas))
|
||||
->values()
|
||||
->all();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, int>
|
||||
*/
|
||||
private function placementServerIds(Service $service): array
|
||||
{
|
||||
$configured = collect($service->config['server_ids'] ?? [])
|
||||
->map(fn (mixed $serverId): int => (int) $serverId)
|
||||
->filter()
|
||||
->unique()
|
||||
->values()
|
||||
->all();
|
||||
|
||||
if ($configured !== []) {
|
||||
return $configured;
|
||||
}
|
||||
|
||||
$existing = $service->replicas
|
||||
->pluck('server_id')
|
||||
->filter()
|
||||
->unique()
|
||||
->values()
|
||||
->all();
|
||||
|
||||
if ($existing !== []) {
|
||||
return $existing;
|
||||
}
|
||||
|
||||
return $service->server_id ? [(int) $service->server_id] : [];
|
||||
}
|
||||
|
||||
private function createGatewayOperations(Operation $parent): void
|
||||
{
|
||||
$this->environment->loadMissing('attachments.service.replicas', 'attachments.serviceSlice');
|
||||
|
||||
foreach ($this->environment->attachments->where('role', EnvironmentAttachmentRole::GATEWAY) as $attachment) {
|
||||
$target = $attachment->serviceSlice ?: $this->environment;
|
||||
|
||||
$sliceConfigure = $target->operations()->create([
|
||||
'parent_id' => $parent->id,
|
||||
'kind' => OperationKind::SLICE_CONFIGURE,
|
||||
'status' => OperationStatus::PENDING,
|
||||
]);
|
||||
$sliceConfigure->steps()->create([
|
||||
'name' => 'Configure Caddy route',
|
||||
'order' => 1,
|
||||
'status' => OperationStatus::PENDING,
|
||||
'script' => $this->configureCaddyRouteScript($attachment),
|
||||
]);
|
||||
|
||||
$gatewayCutover = $this->environment->operations()->create([
|
||||
'parent_id' => $parent->id,
|
||||
'kind' => OperationKind::GATEWAY_CUTOVER,
|
||||
'status' => OperationStatus::PENDING,
|
||||
]);
|
||||
|
||||
foreach ($this->gatewayCutoverSteps($attachment) as $index => $step) {
|
||||
$gatewayCutover->steps()->create([
|
||||
'name' => $step['name'],
|
||||
'order' => $index + 1,
|
||||
'status' => OperationStatus::PENDING,
|
||||
'script' => $step['script'],
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, array{name: string, script: string}>
|
||||
*/
|
||||
private function serviceDeployScripts(Service $service, string $commitSha, string $imageDigest): array
|
||||
{
|
||||
$servicePath = $this->servicePath($service);
|
||||
$composePath = "{$servicePath}/compose.yml";
|
||||
$serviceKey = $this->serviceKey($service);
|
||||
|
||||
$steps = [
|
||||
[
|
||||
'name' => 'Resolve target commit',
|
||||
'script' => implode("\n", [
|
||||
"mkdir -p {$servicePath}",
|
||||
'printf %s '.escapeshellarg($commitSha)." > {$servicePath}/REVISION",
|
||||
]),
|
||||
],
|
||||
[
|
||||
'name' => 'Create or reuse build artifact',
|
||||
'script' => 'printf %s '.escapeshellarg($imageDigest)." > {$servicePath}/IMAGE_DIGEST",
|
||||
],
|
||||
[
|
||||
'name' => 'Render Compose files',
|
||||
'script' => $this->composeUploadScript($service),
|
||||
],
|
||||
];
|
||||
|
||||
foreach ($service->driver()->preSwitchSteps() as $step) {
|
||||
$steps[] = [
|
||||
'name' => $step->name,
|
||||
'script' => $step->getScriptTemplate(),
|
||||
];
|
||||
}
|
||||
|
||||
if (($service->config['migration_timing'] ?? 'pre_switch') === 'pre_switch') {
|
||||
$steps[] = [
|
||||
'name' => 'Run migrations',
|
||||
'script' => app(BuildMigrationScript::class)->execute($service),
|
||||
];
|
||||
}
|
||||
|
||||
$steps = [
|
||||
...$steps,
|
||||
[
|
||||
'name' => 'Deploy replicas',
|
||||
'script' => "docker compose -f {$composePath} up -d --scale {$serviceKey}=".max(1, $service->desired_replicas),
|
||||
],
|
||||
[
|
||||
'name' => 'Health check replicas',
|
||||
'script' => "docker compose -f {$composePath} ps --status running {$serviceKey}",
|
||||
],
|
||||
];
|
||||
|
||||
if (($service->config['migration_timing'] ?? 'pre_switch') === 'post_switch') {
|
||||
$steps[] = [
|
||||
'name' => 'Run migrations',
|
||||
'script' => app(BuildMigrationScript::class)->execute($service),
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
...$steps,
|
||||
[
|
||||
'name' => 'Drain old replicas',
|
||||
'script' => "docker ps --filter 'label=keystone.service_id={$service->id}' --filter 'label=keystone.draining=true' --format '{{.ID}}' | xargs -r docker stop",
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, array{name: string, script: string}>
|
||||
*/
|
||||
private function replicaDeployScripts(Service $service, int $replica, ?string $imageReference = null): array
|
||||
{
|
||||
$composePath = $this->servicePath($service).'/compose.yml';
|
||||
$project = "keystone_service_{$service->id}_replica_{$replica}";
|
||||
$serviceKey = $this->serviceKey($service);
|
||||
|
||||
$steps = [];
|
||||
|
||||
if ($imageReference && $service->available_image_digest) {
|
||||
$steps[] = [
|
||||
'name' => "Pull image for replica {$replica}",
|
||||
'script' => 'docker pull '.escapeshellarg($imageReference.'@'.$service->available_image_digest),
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
...$steps,
|
||||
[
|
||||
'name' => "Render replica {$replica}",
|
||||
'script' => "docker compose -p {$project} -f {$composePath} config --quiet",
|
||||
],
|
||||
[
|
||||
'name' => "Start replica {$replica}",
|
||||
'script' => implode("\n", [
|
||||
"docker compose -p {$project} -f {$composePath} up -d {$serviceKey}",
|
||||
"container_id=$(docker compose -p {$project} -f {$composePath} ps -q {$serviceKey})",
|
||||
'printf "container_id=%s\n" "$container_id"',
|
||||
]),
|
||||
],
|
||||
[
|
||||
'name' => "Health check replica {$replica}",
|
||||
'script' => implode("\n", [
|
||||
"docker compose -p {$project} -f {$composePath} ps --status running {$serviceKey}",
|
||||
"container_id=$(docker compose -p {$project} -f {$composePath} ps -q {$serviceKey})",
|
||||
'health_status=$(docker inspect --format "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}" "$container_id")',
|
||||
'printf "health_status=%s\n" "$health_status"',
|
||||
]),
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
private function composeUploadScript(Service $service): string
|
||||
{
|
||||
$servicePath = $this->servicePath($service);
|
||||
|
||||
try {
|
||||
$renderer = app(ComposeRenderer::class);
|
||||
$compose = $renderer->render($service);
|
||||
$env = $renderer->renderEnvironmentFile($service);
|
||||
} catch (InvalidArgumentException) {
|
||||
$compose = "services:\n {$this->serviceKey($service)}:\n image: \"{$service->available_image_digest}\"\n";
|
||||
$env = '';
|
||||
}
|
||||
|
||||
return implode("\n", [
|
||||
"mkdir -p {$servicePath}",
|
||||
'printf %s '.escapeshellarg(base64_encode($compose))." | base64 -d > {$servicePath}/compose.yml",
|
||||
'printf %s '.escapeshellarg(base64_encode($env))." | base64 -d > {$servicePath}/.env",
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, array{name: string, script: string}>
|
||||
*/
|
||||
private function gatewayCutoverSteps(EnvironmentAttachment $attachment): array
|
||||
{
|
||||
$containerName = $attachment->service->replicas()->first()?->container_name;
|
||||
$reloadCommand = $containerName
|
||||
? 'docker exec '.escapeshellarg($containerName).' caddy reload --config /etc/caddy/Caddyfile'
|
||||
: "docker compose -f /home/keystone/services/{$attachment->service_id}/compose.yml exec -T {$this->serviceKey($attachment->service)} caddy reload --config /etc/caddy/Caddyfile";
|
||||
|
||||
return [
|
||||
[
|
||||
'name' => 'Validate Caddy route configuration',
|
||||
'script' => 'test -s /home/keystone/gateway/Caddyfile',
|
||||
],
|
||||
[
|
||||
'name' => 'Reload Caddy',
|
||||
'script' => $reloadCommand,
|
||||
],
|
||||
[
|
||||
'name' => 'Verify new upstreams are reachable',
|
||||
'script' => 'curl --fail --silent --show-error http://127.0.0.1/ >/dev/null || true',
|
||||
],
|
||||
[
|
||||
'name' => 'Drain old upstreams',
|
||||
'script' => implode("\n", [
|
||||
"docker ps --filter 'label=keystone.environment_id={$this->environment->id}' --filter 'label=keystone.draining=true' --format '{{.ID}}' | xargs -r docker stop --time 30",
|
||||
]),
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
private function configureCaddyRouteScript(EnvironmentAttachment $attachment): string
|
||||
{
|
||||
$route = $attachment->serviceSlice?->name ?? $this->environment->name;
|
||||
$upstreams = $this->gatewayUpstreams($attachment);
|
||||
|
||||
return implode("\n", [
|
||||
'mkdir -p /home/keystone/gateway/Caddyfile.d',
|
||||
"cat > /home/keystone/gateway/Caddyfile.d/{$attachment->id}.caddy <<'KEYSTONE_CADDY_ROUTE'",
|
||||
"{$route} {",
|
||||
' reverse_proxy '.implode(' ', $upstreams),
|
||||
'}',
|
||||
'KEYSTONE_CADDY_ROUTE',
|
||||
'cat /home/keystone/gateway/Caddyfile.d/*.caddy > /home/keystone/gateway/Caddyfile',
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, string>
|
||||
*/
|
||||
private function gatewayUpstreams(EnvironmentAttachment $attachment): array
|
||||
{
|
||||
$gatewayReplica = $attachment->service->replicas()->first();
|
||||
|
||||
return $this->environment->services()
|
||||
->where('type', \App\Enums\ServiceType::LARAVEL)
|
||||
->get()
|
||||
->filter(fn (Service $service): bool => in_array('web', $service->process_roles ?? [], true))
|
||||
->flatMap(function (Service $service) use ($gatewayReplica) {
|
||||
return $service->replicas
|
||||
->map(function (ServiceReplica $replica) use ($gatewayReplica) {
|
||||
$endpoint = app(RegisterServiceEndpoint::class)->execute(
|
||||
replica: $replica,
|
||||
consumerReplica: $gatewayReplica,
|
||||
allowPublicFallback: false,
|
||||
);
|
||||
|
||||
return [
|
||||
'priority' => $endpoint->priority,
|
||||
'target' => $this->endpointTarget($endpoint->scope, $endpoint->hostname, $endpoint->port),
|
||||
];
|
||||
});
|
||||
})
|
||||
->sortBy('priority')
|
||||
->pluck('target')
|
||||
->values()
|
||||
->whenEmpty(fn ($targets) => $targets->push('web:80'))
|
||||
->all();
|
||||
}
|
||||
|
||||
private function endpointTarget(ServiceEndpointScope $scope, string $hostname, int $port): string
|
||||
{
|
||||
return $scope === ServiceEndpointScope::DOCKER_NETWORK
|
||||
? $hostname.':'.$port
|
||||
: 'http://'.$hostname.':'.$port;
|
||||
}
|
||||
|
||||
private function dispatchChildOperations(Operation $operation): void
|
||||
{
|
||||
$operation->update(['status' => OperationStatus::IN_PROGRESS]);
|
||||
|
||||
$operation->children()
|
||||
->with('steps')
|
||||
->orderBy('id')
|
||||
->get()
|
||||
->first(fn (Operation $child): bool => $child->steps->isNotEmpty())
|
||||
?->steps
|
||||
->sortBy('order')
|
||||
->first()
|
||||
?->dispatchJob();
|
||||
}
|
||||
|
||||
private function servicePath(Service $service): string
|
||||
{
|
||||
return "/home/keystone/services/{$service->id}";
|
||||
}
|
||||
|
||||
private function serviceKey(Service $service): string
|
||||
{
|
||||
return str($service->name)->slug('_')->value() ?: 'service';
|
||||
}
|
||||
|
||||
private function defaultInternalPort(Service $service): int
|
||||
{
|
||||
return match ($service->type) {
|
||||
\App\Enums\ServiceType::POSTGRES => 5432,
|
||||
\App\Enums\ServiceType::VALKEY => 6379,
|
||||
\App\Enums\ServiceType::CADDY,
|
||||
\App\Enums\ServiceType::LARAVEL => 80,
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user