harry/keystone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement Keystone environment deployments

Browse Source
This commit is contained in:
Harry Bayliss
2026-05-13 16:11:23 +01:00
parent 65d3142d03
commit aa680b25fd
175 changed files with 10258 additions and 740 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
app/Jobs/Applications/DeployApplication.php
View File

@@ -1,59 +0,0 @@
<?php
namespace App\Jobs\Applications;
use App\Enums\DeploymentStatus;
use App\Models\Application;
use App\Models\Deployment;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Queue\Queueable;
class DeployApplication implements ShouldQueue
{
use Queueable;
protected Deployment $deployment;
public function __construct(
public Application $application,
) {
//
}
public function handle(): void
{
$this->deployment = $this->application->deployments()->create([
'status' => DeploymentStatus::PENDING,
]);
foreach ($this->application->instances as $instance) {
$step = $this->deployment->steps()->create([
'name' => "Deploy to {$instance->server->name}",
'order' => $instance->id,
'status' => DeploymentStatus::PENDING,
'script' => $this->getDeploymentScript($instance),
'secrets' => [],
]);
$step->dispatchJob();
}
}
protected function getDeploymentScript($instance): string
{
return "#!/bin/bash\n" .
"cd /opt/apps/{$this->application->name}-{$instance->id}\n" .
"git fetch origin\n" .
"git checkout {$instance->branch}\n" .
"git pull origin {$instance->branch}\n";
}
public function failed(\Throwable $exception): void
{
if (isset($this->deployment)) {
$this->deployment->update([
'status' => DeploymentStatus::FAILED,
]);
}
}
}

497
app/Jobs/Environments/DeployEnvironment.php Normal file
View File

@@ -0,0 +1,497 @@
<?php
namespace App\Jobs\Environments;
use App\Actions\Environments\BuildApplicationArtifact;
use App\Actions\Environments\BuildMigrationScript;
use App\Actions\Environments\PlanBuildArtifact;
use App\Actions\Environments\PlanEnvironmentDeployment;
use App\Actions\Environments\ResolveEnvironmentCommit;
use App\Actions\Services\RegisterServiceEndpoint;
use App\Enums\EnvironmentAttachmentRole;
use App\Enums\OperationKind;
use App\Enums\OperationStatus;
use App\Enums\ServiceEndpointScope;
use App\Models\Environment;
use App\Models\EnvironmentAttachment;
use App\Models\Operation;
use App\Models\Service;
use App\Models\ServiceReplica;
use App\Services\Compose\ComposeRenderer;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Queue\Queueable;
use InvalidArgumentException;
use RuntimeException;
class DeployEnvironment implements ShouldQueue
{
use Queueable;
public function __construct(
public Environment $environment,
) {
//
}
public function handle(): void
{
$plan = app(PlanEnvironmentDeployment::class)->execute($this->environment);
if ($plan->requiresRegistry) {
throw new RuntimeException('A registry is required before deploying this environment across multiple servers.');
}
if ($plan->blockers !== []) {
throw new RuntimeException($plan->blockers[0]);
}
$operation = $this->environment->operations()->create([
'kind' => OperationKind::ENVIRONMENT_DEPLOY,
'status' => OperationStatus::PENDING,
'started_at' => now(),
]);
$commitSha = app(ResolveEnvironmentCommit::class)->execute($this->environment);
$services = $this->servicesNeedingDeployment($plan->services, $commitSha);
if ($services === []) {
$operation->update([
'status' => OperationStatus::COMPLETED,
'finished_at' => now(),
]);
return;
}
$artifact = app(PlanBuildArtifact::class)->execute($this->environment, $commitSha);
$artifact = app(BuildApplicationArtifact::class)->execute($artifact, $operation);
foreach ($services as $service) {
$service->update([
'available_image_digest' => $artifact->image_digest,
'desired_revision' => $commitSha,
]);
$child = $service->operations()->create([
'parent_id' => $operation->id,
'kind' => OperationKind::SERVICE_DEPLOY,
'status' => OperationStatus::PENDING,
]);
$this->createServiceDeploySteps($child, $service, $commitSha, $artifact->image_digest);
$this->createReplicaDeployOperations($child, $service, $artifact->registry_ref);
}
$this->createGatewayOperations($operation);
$this->dispatchChildOperations($operation);
}
/**
* @param array<int, Service> $services
* @return array<int, Service>
*/
private function servicesNeedingDeployment(array $services, string $commitSha): array
{
return collect($services)
->filter(fn (Service $service): bool => $service->desired_revision !== $commitSha || ! $service->available_image_digest)
->values()
->all();
}
private function createServiceDeploySteps(Operation $operation, Service $service, string $commitSha, string $imageDigest): void
{
foreach ($this->serviceDeployScripts($service, $commitSha, $imageDigest) as $index => $step) {
$operation->steps()->create([
'name' => $step['name'],
'order' => $index + 1,
'status' => OperationStatus::PENDING,
'script' => $step['script'],
]);
}
}
private function createReplicaDeployOperations(Operation $parent, Service $service, ?string $imageReference = null): void
{
$replicas = $this->ensureServiceReplicas($service);
for ($replica = 1; $replica <= max(1, $service->desired_replicas); $replica++) {
$serviceReplica = $replicas[$replica - 1] ?? null;
$target = $serviceReplica ?: $service;
$operation = $target->operations()->create([
'parent_id' => $parent->id,
'kind' => OperationKind::REPLICA_DEPLOY,
'status' => OperationStatus::PENDING,
]);
$serviceReplica?->update([
'operation_id' => $operation->id,
'image_digest' => $service->available_image_digest,
'status' => 'pending',
'health_status' => 'unknown',
]);
foreach ($this->replicaDeployScripts($service, $replica, $imageReference) as $index => $step) {
$operation->steps()->create([
'name' => $step['name'],
'order' => $index + 1,
'status' => OperationStatus::PENDING,
'script' => $step['script'],
]);
}
}
}
/**
* @return array<int, ServiceReplica>
*/
private function ensureServiceReplicas(Service $service): array
{
$service->loadMissing('replicas');
$serverIds = $this->placementServerIds($service);
if ($service->replicas->count() < $service->desired_replicas && $serverIds !== []) {
for ($index = $service->replicas->count() + 1; $index <= $service->desired_replicas; $index++) {
$service->replicas()->create([
'server_id' => $serverIds[($index - 1) % count($serverIds)],
'container_name' => "keystone-service-{$service->id}-{$index}",
'internal_host' => "keystone-service-{$service->id}",
'internal_port' => $this->defaultInternalPort($service),
'status' => 'pending',
'health_status' => 'unknown',
'config' => [],
]);
}
$service->load('replicas');
}
return $service->replicas
->take(max(1, $service->desired_replicas))
->values()
->all();
}
/**
* @return array<int, int>
*/
private function placementServerIds(Service $service): array
{
$configured = collect($service->config['server_ids'] ?? [])
->map(fn (mixed $serverId): int => (int) $serverId)
->filter()
->unique()
->values()
->all();
if ($configured !== []) {
return $configured;
}
$existing = $service->replicas
->pluck('server_id')
->filter()
->unique()
->values()
->all();
if ($existing !== []) {
return $existing;
}
return $service->server_id ? [(int) $service->server_id] : [];
}
private function createGatewayOperations(Operation $parent): void
{
$this->environment->loadMissing('attachments.service.replicas', 'attachments.serviceSlice');
foreach ($this->environment->attachments->where('role', EnvironmentAttachmentRole::GATEWAY) as $attachment) {
$target = $attachment->serviceSlice ?: $this->environment;
$sliceConfigure = $target->operations()->create([
'parent_id' => $parent->id,
'kind' => OperationKind::SLICE_CONFIGURE,
'status' => OperationStatus::PENDING,
]);
$sliceConfigure->steps()->create([
'name' => 'Configure Caddy route',
'order' => 1,
'status' => OperationStatus::PENDING,
'script' => $this->configureCaddyRouteScript($attachment),
]);
$gatewayCutover = $this->environment->operations()->create([
'parent_id' => $parent->id,
'kind' => OperationKind::GATEWAY_CUTOVER,
'status' => OperationStatus::PENDING,
]);
foreach ($this->gatewayCutoverSteps($attachment) as $index => $step) {
$gatewayCutover->steps()->create([
'name' => $step['name'],
'order' => $index + 1,
'status' => OperationStatus::PENDING,
'script' => $step['script'],
]);
}
}
}
/**
* @return array<int, array{name: string, script: string}>
*/
private function serviceDeployScripts(Service $service, string $commitSha, string $imageDigest): array
{
$servicePath = $this->servicePath($service);
$composePath = "{$servicePath}/compose.yml";
$serviceKey = $this->serviceKey($service);
$steps = [
[
'name' => 'Resolve target commit',
'script' => implode("\n", [
"mkdir -p {$servicePath}",
'printf %s '.escapeshellarg($commitSha)." > {$servicePath}/REVISION",
]),
],
[
'name' => 'Create or reuse build artifact',
'script' => 'printf %s '.escapeshellarg($imageDigest)." > {$servicePath}/IMAGE_DIGEST",
],
[
'name' => 'Render Compose files',
'script' => $this->composeUploadScript($service),
],
];
foreach ($service->driver()->preSwitchSteps() as $step) {
$steps[] = [
'name' => $step->name,
'script' => $step->getScriptTemplate(),
];
}
if (($service->config['migration_timing'] ?? 'pre_switch') === 'pre_switch') {
$steps[] = [
'name' => 'Run migrations',
'script' => app(BuildMigrationScript::class)->execute($service),
];
}
$steps = [
...$steps,
[
'name' => 'Deploy replicas',
'script' => "docker compose -f {$composePath} up -d --scale {$serviceKey}=".max(1, $service->desired_replicas),
],
[
'name' => 'Health check replicas',
'script' => "docker compose -f {$composePath} ps --status running {$serviceKey}",
],
];
if (($service->config['migration_timing'] ?? 'pre_switch') === 'post_switch') {
$steps[] = [
'name' => 'Run migrations',
'script' => app(BuildMigrationScript::class)->execute($service),
];
}
return [
...$steps,
[
'name' => 'Drain old replicas',
'script' => "docker ps --filter 'label=keystone.service_id={$service->id}' --filter 'label=keystone.draining=true' --format '{{.ID}}' | xargs -r docker stop",
],
];
}
/**
* @return array<int, array{name: string, script: string}>
*/
private function replicaDeployScripts(Service $service, int $replica, ?string $imageReference = null): array
{
$composePath = $this->servicePath($service).'/compose.yml';
$project = "keystone_service_{$service->id}_replica_{$replica}";
$serviceKey = $this->serviceKey($service);
$steps = [];
if ($imageReference && $service->available_image_digest) {
$steps[] = [
'name' => "Pull image for replica {$replica}",
'script' => 'docker pull '.escapeshellarg($imageReference.'@'.$service->available_image_digest),
];
}
return [
...$steps,
[
'name' => "Render replica {$replica}",
'script' => "docker compose -p {$project} -f {$composePath} config --quiet",
],
[
'name' => "Start replica {$replica}",
'script' => implode("\n", [
"docker compose -p {$project} -f {$composePath} up -d {$serviceKey}",
"container_id=$(docker compose -p {$project} -f {$composePath} ps -q {$serviceKey})",
'printf "container_id=%s\n" "$container_id"',
]),
],
[
'name' => "Health check replica {$replica}",
'script' => implode("\n", [
"docker compose -p {$project} -f {$composePath} ps --status running {$serviceKey}",
"container_id=$(docker compose -p {$project} -f {$composePath} ps -q {$serviceKey})",
'health_status=$(docker inspect --format "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}" "$container_id")',
'printf "health_status=%s\n" "$health_status"',
]),
],
];
}
private function composeUploadScript(Service $service): string
{
$servicePath = $this->servicePath($service);
try {
$renderer = app(ComposeRenderer::class);
$compose = $renderer->render($service);
$env = $renderer->renderEnvironmentFile($service);
} catch (InvalidArgumentException) {
$compose = "services:\n {$this->serviceKey($service)}:\n image: \"{$service->available_image_digest}\"\n";
$env = '';
}
return implode("\n", [
"mkdir -p {$servicePath}",
'printf %s '.escapeshellarg(base64_encode($compose))." | base64 -d > {$servicePath}/compose.yml",
'printf %s '.escapeshellarg(base64_encode($env))." | base64 -d > {$servicePath}/.env",
]);
}
/**
* @return array<int, array{name: string, script: string}>
*/
private function gatewayCutoverSteps(EnvironmentAttachment $attachment): array
{
$containerName = $attachment->service->replicas()->first()?->container_name;
$reloadCommand = $containerName
? 'docker exec '.escapeshellarg($containerName).' caddy reload --config /etc/caddy/Caddyfile'
: "docker compose -f /home/keystone/services/{$attachment->service_id}/compose.yml exec -T {$this->serviceKey($attachment->service)} caddy reload --config /etc/caddy/Caddyfile";
return [
[
'name' => 'Validate Caddy route configuration',
'script' => 'test -s /home/keystone/gateway/Caddyfile',
],
[
'name' => 'Reload Caddy',
'script' => $reloadCommand,
],
[
'name' => 'Verify new upstreams are reachable',
'script' => 'curl --fail --silent --show-error http://127.0.0.1/ >/dev/null || true',
],
[
'name' => 'Drain old upstreams',
'script' => implode("\n", [
"docker ps --filter 'label=keystone.environment_id={$this->environment->id}' --filter 'label=keystone.draining=true' --format '{{.ID}}' | xargs -r docker stop --time 30",
]),
],
];
}
private function configureCaddyRouteScript(EnvironmentAttachment $attachment): string
{
$route = $attachment->serviceSlice?->name ?? $this->environment->name;
$upstreams = $this->gatewayUpstreams($attachment);
return implode("\n", [
'mkdir -p /home/keystone/gateway/Caddyfile.d',
"cat > /home/keystone/gateway/Caddyfile.d/{$attachment->id}.caddy <<'KEYSTONE_CADDY_ROUTE'",
"{$route} {",
' reverse_proxy '.implode(' ', $upstreams),
'}',
'KEYSTONE_CADDY_ROUTE',
'cat /home/keystone/gateway/Caddyfile.d/*.caddy > /home/keystone/gateway/Caddyfile',
]);
}
/**
* @return array<int, string>
*/
private function gatewayUpstreams(EnvironmentAttachment $attachment): array
{
$gatewayReplica = $attachment->service->replicas()->first();
return $this->environment->services()
->where('type', \App\Enums\ServiceType::LARAVEL)
->get()
->filter(fn (Service $service): bool => in_array('web', $service->process_roles ?? [], true))
->flatMap(function (Service $service) use ($gatewayReplica) {
return $service->replicas
->map(function (ServiceReplica $replica) use ($gatewayReplica) {
$endpoint = app(RegisterServiceEndpoint::class)->execute(
replica: $replica,
consumerReplica: $gatewayReplica,
allowPublicFallback: false,
);
return [
'priority' => $endpoint->priority,
'target' => $this->endpointTarget($endpoint->scope, $endpoint->hostname, $endpoint->port),
];
});
})
->sortBy('priority')
->pluck('target')
->values()
->whenEmpty(fn ($targets) => $targets->push('web:80'))
->all();
}
private function endpointTarget(ServiceEndpointScope $scope, string $hostname, int $port): string
{
return $scope === ServiceEndpointScope::DOCKER_NETWORK
? $hostname.':'.$port
: 'http://'.$hostname.':'.$port;
}
private function dispatchChildOperations(Operation $operation): void
{
$operation->update(['status' => OperationStatus::IN_PROGRESS]);
$operation->children()
->with('steps')
->orderBy('id')
->get()
->first(fn (Operation $child): bool => $child->steps->isNotEmpty())
?->steps
->sortBy('order')
->first()
?->dispatchJob();
}
private function servicePath(Service $service): string
{
return "/home/keystone/services/{$service->id}";
}
private function serviceKey(Service $service): string
{
return str($service->name)->slug('_')->value() ?: 'service';
}
private function defaultInternalPort(Service $service): int
{
return match ($service->type) {
\App\Enums\ServiceType::POSTGRES => 5432,
\App\Enums\ServiceType::VALKEY => 6379,
\App\Enums\ServiceType::CADDY,
\App\Enums\ServiceType::LARAVEL => 80,
};
}
}

73
app/Jobs/Services/DeployService.php
View File

@@ -2,18 +2,23 @@
namespace App\Jobs\Services;
use App\Enums\DeploymentStatus;
use App\Actions\Services\ResolveServiceImageDigest;
use App\Data\Operations\PlannedStep;
use App\Enums\OperationKind;
use App\Enums\OperationStatus;
use App\Enums\ServiceStatus;
use App\Models\Deployment;
use App\Models\Operation;
use App\Models\Service;
use App\Services\Compose\ComposeRenderer;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Queue\Queueable;
use InvalidArgumentException;
class DeployService implements ShouldQueue
{
use Queueable;
protected Deployment $deployment;
protected Operation $operation;
public function __construct(
public Service $service,
@@ -24,20 +29,26 @@ class DeployService implements ShouldQueue
public function handle(): void
{
$driver = $this->service->driver();
$this->service->forceFill([
'available_image_digest' => app(ResolveServiceImageDigest::class)->execute($this->service),
])->save();
$this->service->update([
'status' => ServiceStatus::INSTALLING,
]);
$this->deployment = $this->service->deployments()->create([
'status' => DeploymentStatus::PENDING,
$this->operation = $this->service->operations()->create([
'kind' => OperationKind::SERVICE_DEPLOY,
'status' => OperationStatus::PENDING,
]);
$deploymentPlan = $driver->getDeploymentPlan($this->deployment->hash);
foreach ($deploymentPlan->steps as $index => $plannedStep) {
$step = $this->deployment->steps()->create([
$operationPlan = $driver->getOperationPlan($this->operation->hash);
$steps = $this->stepsWithComposeUpload($operationPlan->steps);
foreach ($steps as $index => $plannedStep) {
$step = $this->operation->steps()->create([
'name' => $plannedStep->name,
'order' => $index + 1,
'status' => DeploymentStatus::PENDING,
'script' => $plannedStep->getSafeScript(),
'secrets' => $this->service->credentials,
'status' => OperationStatus::PENDING,
'script' => $plannedStep->getScriptTemplate(),
'secrets' => $plannedStep->secrets(),
]);
if ($index === 0) {
$step->dispatchJob();
@@ -45,11 +56,45 @@ class DeployService implements ShouldQueue
}
}
/**
* @param array<int, \App\Data\Operations\PlannedStep> $steps
* @return array<int, \App\Data\Operations\PlannedStep>
*/
private function stepsWithComposeUpload(array $steps): array
{
try {
$renderer = app(ComposeRenderer::class);
$compose = $renderer->render($this->service);
$env = $renderer->renderEnvironmentFile($this->service);
} catch (InvalidArgumentException) {
return $steps;
}
return [
new PlannedStep(
name: 'Upload Compose file',
script: $this->composeUploadScript($compose, $env),
),
...$steps,
];
}
private function composeUploadScript(string $compose, string $env): string
{
$servicePath = "/home/keystone/services/{$this->service->id}";
return implode("\n", [
"mkdir -p {$servicePath}",
'printf %s '.escapeshellarg(base64_encode($compose))." | base64 -d > {$servicePath}/compose.yml",
'printf %s '.escapeshellarg(base64_encode($env))." | base64 -d > {$servicePath}/.env",
]);
}
public function failed(\Throwable $exception): void
{
if (isset($this->deployment)) {
$this->deployment->update([
'status' => DeploymentStatus::FAILED,
if (isset($this->operation)) {
$this->operation->update([
'status' => OperationStatus::FAILED,
]);
$this->service->update([
'status' => ServiceStatus::ERROR,

244
app/Jobs/Services/RunStep.php
View File

@@ -2,95 +2,251 @@
namespace App\Jobs\Services;
use App\Enums\DeploymentStatus;
use App\Enums\OperationStatus;
use App\Enums\ServiceStatus;
use App\Models\Step;
use App\Models\Environment;
use App\Models\Operation;
use App\Models\OperationStep;
use App\Models\Server;
use App\Models\Service;
use App\Models\ServiceReplica;
use App\Models\ServiceSlice;
use App\Services\Operations\RemoteCommandRunner;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Queue\Queueable;
use Symfony\Component\Process\Process;
class RunStep implements ShouldQueue
{
use Queueable;
public function __construct(
protected Step $step,
protected OperationStep $step,
) {
//
}
public function handle(): void
{
$this->step->load('deployment.target.server');
$this->step->load('operation.target');
$this->step->operation->update([
'status' => OperationStatus::IN_PROGRESS,
]);
$this->step->update([
'status' => DeploymentStatus::IN_PROGRESS,
'status' => OperationStatus::IN_PROGRESS,
'started_at' => now(),
]);
$server = $this->step->deployment->target->server;
$server = $this->targetServer();
$ssh = $server->sshClient()
->onOutput(function ($type, $output) {
if (trim($output) === '') {
return;
}
if ($type === Process::OUT) {
$this->step->update([
'logs' => $this->step->logs . "\n" . trim($output),
]);
} else {
$this->step->update([
'error_logs' => $this->step->error_logs . "\n" . trim($output),
]);
}
});
$result = $ssh->execute($this->step->script);
if (! $result->isSuccessful()) {
$this->step->update([
'status' => DeploymentStatus::FAILED,
'finished_at' => now(),
'error_logs' => $this->step->error_logs . "\n" . trim($result->getErrorOutput()),
]);
try {
$output = app(RemoteCommandRunner::class)->run($server, $this->step->scriptForExecution());
} catch (\Throwable $exception) {
$this->failStep($exception->getMessage());
return;
}
$this->step->update([
'status' => DeploymentStatus::COMPLETED,
'status' => OperationStatus::COMPLETED,
'finished_at' => now(),
'logs' => trim($this->step->logs."\n".$output),
'secrets' => null,
]);
$this->captureRuntimeState();
// Dispatch the next step if available
if ($nextStep = $this->step->deployment->steps()->where('order', '>', $this->step->order)->orderBy('order', 'asc')->first()) {
if ($nextStep = $this->step->operation->steps()->where('order', '>', $this->step->order)->orderBy('order', 'asc')->first()) {
$nextStep->dispatchJob();
} elseif ($this->dispatchNextChildOperation($this->step->operation)) {
return;
} else {
$this->step->deployment->update([
'status' => DeploymentStatus::COMPLETED,
'finished_at' => now(),
]);
$this->step->deployment->target->update([
$this->completeOperation($this->step->operation);
$this->dispatchNextOperationAfter($this->step->operation);
}
}
private function captureRuntimeState(): void
{
$target = $this->step->operation->target;
if (! $target instanceof ServiceReplica) {
return;
}
$state = $this->step->refresh()->capturedRuntimeState();
if ($state !== []) {
$target->update($state);
}
}
private function markTargetCompleted(): void
{
$target = $this->step->operation->target;
if ($target instanceof Service) {
$target->update([
'status' => ServiceStatus::RUNNING,
]);
return;
}
if ($target instanceof ServiceReplica) {
$target->update([
'status' => 'running',
'health_status' => $target->health_status === 'unknown' ? 'healthy' : $target->health_status,
]);
}
}
private function completeOperation(Operation $operation): void
{
$operation->update([
'status' => OperationStatus::COMPLETED,
'finished_at' => now(),
]);
if ($operation->is($this->step->operation)) {
$this->markTargetCompleted();
}
}
private function dispatchNextChildOperation(Operation $operation): bool
{
$child = $operation->children()
->where('status', OperationStatus::PENDING)
->orderBy('id')
->get()
->first(fn (Operation $child): bool => $child->steps()->exists());
if (! $child) {
return false;
}
$child->steps()->orderBy('order')->first()?->dispatchJob();
return true;
}
private function dispatchNextOperationAfter(Operation $operation): void
{
$operation->loadMissing('parent');
if (! $operation->parent_id) {
return;
}
$nextSibling = $operation->parent
?->children()
->where('id', '>', $operation->id)
->where('status', OperationStatus::PENDING)
->orderBy('id')
->first();
if ($nextSibling) {
$nextSibling->steps()->orderBy('order')->first()?->dispatchJob();
return;
}
$parent = $operation->parent;
if ($parent && $parent->status === OperationStatus::IN_PROGRESS) {
$this->completeOperation($parent);
$this->dispatchNextOperationAfter($parent);
}
}
private function targetServer(): Server
{
$target = $this->step->operation->target;
$server = match (true) {
$target instanceof ServiceReplica => $target->server,
$target instanceof Service => $target->replicas()->with('server')->first()?->server ?: $target->server,
$target instanceof ServiceSlice => $target->service->replicas()->with('server')->first()?->server ?: $target->service->server,
$target instanceof Environment => $target->services()->with(['server', 'replicas.server'])->get()
->flatMap(fn (Service $service) => $service->replicas->pluck('server')->filter())
->first() ?: $target->services()->with('server')->get()->pluck('server')->filter()->first(),
default => null,
};
if (! $server instanceof Server) {
throw new \RuntimeException('Operation target does not have a server for SSH execution.');
}
return $server;
}
public function failed(\Throwable $exception): void
{
$this->failStep($exception->getMessage());
}
private function failStep(string $message): void
{
$this->step->update([
'status' => DeploymentStatus::FAILED,
'status' => OperationStatus::FAILED,
'finished_at' => now(),
'error_logs' => $this->step->error_logs . "\n" . trim($exception->getMessage()),
'error_logs' => $this->step->error_logs."\n".trim($message),
]);
$this->step->deployment->steps()->where('order', '>', $this->step->order)->update([
'status' => DeploymentStatus::CANCELLED,
$this->step->operation->steps()->where('order', '>', $this->step->order)->update([
'status' => OperationStatus::CANCELLED,
]);
$this->step->deployment->update([
'status' => DeploymentStatus::FAILED,
$this->step->operation->update([
'status' => OperationStatus::FAILED,
'finished_at' => now(),
]);
$this->cancelDescendants($this->step->operation);
$this->cancelPendingSiblingsAndAncestors($this->step->operation);
}
private function cancelDescendants(Operation $operation): void
{
$operation->children()->with('children')->get()->each(function (Operation $child): void {
$child->steps()->where('status', OperationStatus::PENDING)->update([
'status' => OperationStatus::CANCELLED,
]);
$child->update([
'status' => OperationStatus::CANCELLED,
'finished_at' => now(),
]);
$this->cancelDescendants($child);
});
}
private function cancelPendingSiblingsAndAncestors(Operation $operation): void
{
$operation->loadMissing('parent');
if (! $operation->parent) {
return;
}
$operation->parent->children()
->where('id', '!=', $operation->id)
->whereIn('status', [OperationStatus::PENDING, OperationStatus::IN_PROGRESS])
->get()
->each(function (Operation $sibling): void {
$sibling->steps()->where('status', OperationStatus::PENDING)->update([
'status' => OperationStatus::CANCELLED,
]);
$sibling->update([
'status' => OperationStatus::CANCELLED,
'finished_at' => now(),
]);
$this->cancelDescendants($sibling);
});
$operation->parent->update([
'status' => OperationStatus::FAILED,
'finished_at' => now(),
]);
$this->cancelPendingSiblingsAndAncestors($operation->parent);
}
}