step->load('operation.target'); $this->step->operation->update([ 'status' => OperationStatus::IN_PROGRESS, ]); $this->step->update([ 'status' => OperationStatus::IN_PROGRESS, 'started_at' => now(), ]); $server = $this->targetServer(); try { $output = app(RemoteCommandRunner::class)->run($server, $this->step->scriptForExecution()); } catch (\Throwable $exception) { $this->failStep($exception->getMessage()); return; } $this->step->update([ 'status' => OperationStatus::COMPLETED, 'finished_at' => now(), 'logs' => trim($this->step->logs."\n".$output), 'secrets' => null, ]); $this->captureRuntimeState(); // Dispatch the next step if available if ($nextStep = $this->step->operation->steps()->where('order', '>', $this->step->order)->orderBy('order', 'asc')->first()) { $nextStep->dispatchJob(); } elseif ($this->dispatchNextChildOperation($this->step->operation)) { return; } else { $this->completeOperation($this->step->operation); $this->dispatchNextOperationAfter($this->step->operation); } } private function captureRuntimeState(): void { $target = $this->step->operation->target; if (! $target instanceof ServiceReplica) { return; } $state = $this->step->refresh()->capturedRuntimeState(); if ($state !== []) { $target->update($state); } } private function markTargetCompleted(): void { $target = $this->step->operation->target; if ($target instanceof Service) { $target->update([ 'status' => ServiceStatus::RUNNING, ]); return; } if ($target instanceof ServiceReplica) { $target->update([ 'status' => 'running', 'health_status' => $target->health_status === 'unknown' ? 'healthy' : $target->health_status, ]); } } private function completeOperation(Operation $operation): void { $operation->update([ 'status' => OperationStatus::COMPLETED, 'finished_at' => now(), ]); if ($operation->is($this->step->operation)) { $this->markTargetCompleted(); } } private function dispatchNextChildOperation(Operation $operation): bool { $child = $operation->children() ->where('status', OperationStatus::PENDING) ->orderBy('id') ->get() ->first(fn (Operation $child): bool => $child->steps()->exists()); if (! $child) { return false; } $child->steps()->orderBy('order')->first()?->dispatchJob(); return true; } private function dispatchNextOperationAfter(Operation $operation): void { $operation->loadMissing('parent'); if (! $operation->parent_id) { return; } $nextSibling = $operation->parent ?->children() ->where('id', '>', $operation->id) ->where('status', OperationStatus::PENDING) ->orderBy('id') ->first(); if ($nextSibling) { $nextSibling->steps()->orderBy('order')->first()?->dispatchJob(); return; } $parent = $operation->parent; if ($parent && $parent->status === OperationStatus::IN_PROGRESS) { $this->completeOperation($parent); $this->dispatchNextOperationAfter($parent); } } private function targetServer(): Server { $target = $this->step->operation->target; $server = match (true) { $target instanceof ServiceReplica => $target->server, $target instanceof Service => $target->replicas()->with('server')->first()?->server ?: $target->server, $target instanceof ServiceSlice => $target->service->replicas()->with('server')->first()?->server ?: $target->service->server, $target instanceof Environment => $target->services()->with(['server', 'replicas.server'])->get() ->flatMap(fn (Service $service) => $service->replicas->pluck('server')->filter()) ->first() ?: $target->services()->with('server')->get()->pluck('server')->filter()->first(), default => null, }; if (! $server instanceof Server) { throw new \RuntimeException('Operation target does not have a server for SSH execution.'); } return $server; } public function failed(\Throwable $exception): void { $this->failStep($exception->getMessage()); } private function failStep(string $message): void { $this->step->update([ 'status' => OperationStatus::FAILED, 'finished_at' => now(), 'error_logs' => $this->step->error_logs."\n".trim($message), ]); $this->step->operation->steps()->where('order', '>', $this->step->order)->update([ 'status' => OperationStatus::CANCELLED, ]); $this->step->operation->update([ 'status' => OperationStatus::FAILED, 'finished_at' => now(), ]); $this->cancelDescendants($this->step->operation); $this->cancelPendingSiblingsAndAncestors($this->step->operation); } private function cancelDescendants(Operation $operation): void { $operation->children()->with('children')->get()->each(function (Operation $child): void { $child->steps()->where('status', OperationStatus::PENDING)->update([ 'status' => OperationStatus::CANCELLED, ]); $child->update([ 'status' => OperationStatus::CANCELLED, 'finished_at' => now(), ]); $this->cancelDescendants($child); }); } private function cancelPendingSiblingsAndAncestors(Operation $operation): void { $operation->loadMissing('parent'); if (! $operation->parent) { return; } $operation->parent->children() ->where('id', '!=', $operation->id) ->whereIn('status', [OperationStatus::PENDING, OperationStatus::IN_PROGRESS]) ->get() ->each(function (Operation $sibling): void { $sibling->steps()->where('status', OperationStatus::PENDING)->update([ 'status' => OperationStatus::CANCELLED, ]); $sibling->update([ 'status' => OperationStatus::CANCELLED, 'finished_at' => now(), ]); $this->cancelDescendants($sibling); }); $operation->parent->update([ 'status' => OperationStatus::FAILED, 'finished_at' => now(), ]); $this->cancelPendingSiblingsAndAncestors($operation->parent); } }