keystone/app/Jobs/Services/RunStep.php

<?php

namespace App\Jobs\Services;

use App\Enums\OperationStatus;
use App\Enums\ServiceStatus;
use App\Models\Environment;
use App\Models\Operation;
use App\Models\OperationStep;
use App\Models\Server;
use App\Models\Service;
use App\Models\ServiceReplica;
use App\Models\ServiceSlice;
use App\Services\Operations\RemoteCommandRunner;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Queue\Queueable;

class RunStep implements ShouldQueue
{
    use Queueable;

    public function __construct(
        protected OperationStep $step,
    ) {
        //
    }

    public function handle(): void
    {
        $this->step->load('operation.target');
        $this->step->operation->update([
            'status' => OperationStatus::IN_PROGRESS,
        ]);
        $this->step->update([
            'status' => OperationStatus::IN_PROGRESS,
            'started_at' => now(),
        ]);

        $server = $this->targetServer();

        try {
            $output = app(RemoteCommandRunner::class)->run($server, $this->step->scriptForExecution());
        } catch (\Throwable $exception) {
            $this->failStep($exception->getMessage());

            return;
        }

        $this->step->update([
            'status' => OperationStatus::COMPLETED,
            'finished_at' => now(),
            'logs' => trim($this->step->logs."\n".$output),
            'secrets' => null,
        ]);
        $this->captureRuntimeState();

        // Dispatch the next step if available
        if ($nextStep = $this->step->operation->steps()->where('order', '>', $this->step->order)->orderBy('order', 'asc')->first()) {
            $nextStep->dispatchJob();
        } elseif ($this->dispatchNextChildOperation($this->step->operation)) {
            return;
        } else {
            $this->completeOperation($this->step->operation);
            $this->dispatchNextOperationAfter($this->step->operation);
        }
    }

    private function captureRuntimeState(): void
    {
        $target = $this->step->operation->target;

        if (! $target instanceof ServiceReplica) {
            return;
        }

        $state = $this->step->refresh()->capturedRuntimeState();

        if ($state !== []) {
            $target->update($state);
        }
    }

    private function markTargetCompleted(): void
    {
        $target = $this->step->operation->target;

        if ($target instanceof Service) {
            $target->update([
                'status' => ServiceStatus::RUNNING,
            ]);

            return;
        }

        if ($target instanceof ServiceReplica) {
            $target->update([
                'status' => 'running',
                'health_status' => $target->health_status === 'unknown' ? 'healthy' : $target->health_status,
            ]);
        }
    }

    private function completeOperation(Operation $operation): void
    {
        $operation->update([
            'status' => OperationStatus::COMPLETED,
            'finished_at' => now(),
        ]);

        if ($operation->is($this->step->operation)) {
            $this->markTargetCompleted();
        }
    }

    private function dispatchNextChildOperation(Operation $operation): bool
    {
        $child = $operation->children()
            ->where('status', OperationStatus::PENDING)
            ->orderBy('id')
            ->get()
            ->first(fn (Operation $child): bool => $child->steps()->exists());

        if (! $child) {
            return false;
        }

        $child->steps()->orderBy('order')->first()?->dispatchJob();

        return true;
    }

    private function dispatchNextOperationAfter(Operation $operation): void
    {
        $operation->loadMissing('parent');

        if (! $operation->parent_id) {
            return;
        }

        $nextSibling = $operation->parent
            ?->children()
            ->where('id', '>', $operation->id)
            ->where('status', OperationStatus::PENDING)
            ->orderBy('id')
            ->first();

        if ($nextSibling) {
            $nextSibling->steps()->orderBy('order')->first()?->dispatchJob();

            return;
        }

        $parent = $operation->parent;

        if ($parent && $parent->status === OperationStatus::IN_PROGRESS) {
            $this->completeOperation($parent);
            $this->dispatchNextOperationAfter($parent);
        }
    }

    private function targetServer(): Server
    {
        $target = $this->step->operation->target;

        $server = match (true) {
            $target instanceof ServiceReplica => $target->server,
            $target instanceof Service => $target->replicas()->with('server')->first()?->server ?: $target->server,
            $target instanceof ServiceSlice => $target->service->replicas()->with('server')->first()?->server ?: $target->service->server,
            $target instanceof Environment => $target->services()->with(['server', 'replicas.server'])->get()
                ->flatMap(fn (Service $service) => $service->replicas->pluck('server')->filter())
                ->first() ?: $target->services()->with('server')->get()->pluck('server')->filter()->first(),
            default => null,
        };

        if (! $server instanceof Server) {
            throw new \RuntimeException('Operation target does not have a server for SSH execution.');
        }

        return $server;
    }

    public function failed(\Throwable $exception): void
    {
        $this->failStep($exception->getMessage());
    }

    private function failStep(string $message): void
    {
        $this->step->update([
            'status' => OperationStatus::FAILED,
            'finished_at' => now(),
            'error_logs' => $this->step->error_logs."\n".trim($message),
        ]);

        $this->step->operation->steps()->where('order', '>', $this->step->order)->update([
            'status' => OperationStatus::CANCELLED,
        ]);

        $this->step->operation->update([
            'status' => OperationStatus::FAILED,
            'finished_at' => now(),
        ]);

        $this->cancelDescendants($this->step->operation);
        $this->cancelPendingSiblingsAndAncestors($this->step->operation);
    }

    private function cancelDescendants(Operation $operation): void
    {
        $operation->children()->with('children')->get()->each(function (Operation $child): void {
            $child->steps()->where('status', OperationStatus::PENDING)->update([
                'status' => OperationStatus::CANCELLED,
            ]);
            $child->update([
                'status' => OperationStatus::CANCELLED,
                'finished_at' => now(),
            ]);
            $this->cancelDescendants($child);
        });
    }

    private function cancelPendingSiblingsAndAncestors(Operation $operation): void
    {
        $operation->loadMissing('parent');

        if (! $operation->parent) {
            return;
        }

        $operation->parent->children()
            ->where('id', '!=', $operation->id)
            ->whereIn('status', [OperationStatus::PENDING, OperationStatus::IN_PROGRESS])
            ->get()
            ->each(function (Operation $sibling): void {
                $sibling->steps()->where('status', OperationStatus::PENDING)->update([
                    'status' => OperationStatus::CANCELLED,
                ]);
                $sibling->update([
                    'status' => OperationStatus::CANCELLED,
                    'finished_at' => now(),
                ]);
                $this->cancelDescendants($sibling);
            });

        $operation->parent->update([
            'status' => OperationStatus::FAILED,
            'finished_at' => now(),
        ]);

        $this->cancelPendingSiblingsAndAncestors($operation->parent);
    }
}