Implement Keystone environment deployments
This commit is contained in:
@@ -2,95 +2,251 @@
|
||||
|
||||
namespace App\Jobs\Services;
|
||||
|
||||
use App\Enums\DeploymentStatus;
|
||||
use App\Enums\OperationStatus;
|
||||
use App\Enums\ServiceStatus;
|
||||
use App\Models\Step;
|
||||
use App\Models\Environment;
|
||||
use App\Models\Operation;
|
||||
use App\Models\OperationStep;
|
||||
use App\Models\Server;
|
||||
use App\Models\Service;
|
||||
use App\Models\ServiceReplica;
|
||||
use App\Models\ServiceSlice;
|
||||
use App\Services\Operations\RemoteCommandRunner;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Queue\Queueable;
|
||||
use Symfony\Component\Process\Process;
|
||||
|
||||
class RunStep implements ShouldQueue
|
||||
{
|
||||
use Queueable;
|
||||
|
||||
public function __construct(
|
||||
protected Step $step,
|
||||
protected OperationStep $step,
|
||||
) {
|
||||
//
|
||||
}
|
||||
|
||||
public function handle(): void
|
||||
{
|
||||
$this->step->load('deployment.target.server');
|
||||
$this->step->load('operation.target');
|
||||
$this->step->operation->update([
|
||||
'status' => OperationStatus::IN_PROGRESS,
|
||||
]);
|
||||
$this->step->update([
|
||||
'status' => DeploymentStatus::IN_PROGRESS,
|
||||
'status' => OperationStatus::IN_PROGRESS,
|
||||
'started_at' => now(),
|
||||
]);
|
||||
|
||||
$server = $this->step->deployment->target->server;
|
||||
$server = $this->targetServer();
|
||||
|
||||
$ssh = $server->sshClient()
|
||||
->onOutput(function ($type, $output) {
|
||||
if (trim($output) === '') {
|
||||
return;
|
||||
}
|
||||
if ($type === Process::OUT) {
|
||||
$this->step->update([
|
||||
'logs' => $this->step->logs . "\n" . trim($output),
|
||||
]);
|
||||
} else {
|
||||
$this->step->update([
|
||||
'error_logs' => $this->step->error_logs . "\n" . trim($output),
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
$result = $ssh->execute($this->step->script);
|
||||
|
||||
if (! $result->isSuccessful()) {
|
||||
$this->step->update([
|
||||
'status' => DeploymentStatus::FAILED,
|
||||
'finished_at' => now(),
|
||||
'error_logs' => $this->step->error_logs . "\n" . trim($result->getErrorOutput()),
|
||||
]);
|
||||
try {
|
||||
$output = app(RemoteCommandRunner::class)->run($server, $this->step->scriptForExecution());
|
||||
} catch (\Throwable $exception) {
|
||||
$this->failStep($exception->getMessage());
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$this->step->update([
|
||||
'status' => DeploymentStatus::COMPLETED,
|
||||
'status' => OperationStatus::COMPLETED,
|
||||
'finished_at' => now(),
|
||||
'logs' => trim($this->step->logs."\n".$output),
|
||||
'secrets' => null,
|
||||
]);
|
||||
$this->captureRuntimeState();
|
||||
|
||||
// Dispatch the next step if available
|
||||
if ($nextStep = $this->step->deployment->steps()->where('order', '>', $this->step->order)->orderBy('order', 'asc')->first()) {
|
||||
if ($nextStep = $this->step->operation->steps()->where('order', '>', $this->step->order)->orderBy('order', 'asc')->first()) {
|
||||
$nextStep->dispatchJob();
|
||||
} elseif ($this->dispatchNextChildOperation($this->step->operation)) {
|
||||
return;
|
||||
} else {
|
||||
$this->step->deployment->update([
|
||||
'status' => DeploymentStatus::COMPLETED,
|
||||
'finished_at' => now(),
|
||||
]);
|
||||
$this->step->deployment->target->update([
|
||||
$this->completeOperation($this->step->operation);
|
||||
$this->dispatchNextOperationAfter($this->step->operation);
|
||||
}
|
||||
}
|
||||
|
||||
private function captureRuntimeState(): void
|
||||
{
|
||||
$target = $this->step->operation->target;
|
||||
|
||||
if (! $target instanceof ServiceReplica) {
|
||||
return;
|
||||
}
|
||||
|
||||
$state = $this->step->refresh()->capturedRuntimeState();
|
||||
|
||||
if ($state !== []) {
|
||||
$target->update($state);
|
||||
}
|
||||
}
|
||||
|
||||
private function markTargetCompleted(): void
|
||||
{
|
||||
$target = $this->step->operation->target;
|
||||
|
||||
if ($target instanceof Service) {
|
||||
$target->update([
|
||||
'status' => ServiceStatus::RUNNING,
|
||||
]);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if ($target instanceof ServiceReplica) {
|
||||
$target->update([
|
||||
'status' => 'running',
|
||||
'health_status' => $target->health_status === 'unknown' ? 'healthy' : $target->health_status,
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
private function completeOperation(Operation $operation): void
|
||||
{
|
||||
$operation->update([
|
||||
'status' => OperationStatus::COMPLETED,
|
||||
'finished_at' => now(),
|
||||
]);
|
||||
|
||||
if ($operation->is($this->step->operation)) {
|
||||
$this->markTargetCompleted();
|
||||
}
|
||||
}
|
||||
|
||||
private function dispatchNextChildOperation(Operation $operation): bool
|
||||
{
|
||||
$child = $operation->children()
|
||||
->where('status', OperationStatus::PENDING)
|
||||
->orderBy('id')
|
||||
->get()
|
||||
->first(fn (Operation $child): bool => $child->steps()->exists());
|
||||
|
||||
if (! $child) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$child->steps()->orderBy('order')->first()?->dispatchJob();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function dispatchNextOperationAfter(Operation $operation): void
|
||||
{
|
||||
$operation->loadMissing('parent');
|
||||
|
||||
if (! $operation->parent_id) {
|
||||
return;
|
||||
}
|
||||
|
||||
$nextSibling = $operation->parent
|
||||
?->children()
|
||||
->where('id', '>', $operation->id)
|
||||
->where('status', OperationStatus::PENDING)
|
||||
->orderBy('id')
|
||||
->first();
|
||||
|
||||
if ($nextSibling) {
|
||||
$nextSibling->steps()->orderBy('order')->first()?->dispatchJob();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$parent = $operation->parent;
|
||||
|
||||
if ($parent && $parent->status === OperationStatus::IN_PROGRESS) {
|
||||
$this->completeOperation($parent);
|
||||
$this->dispatchNextOperationAfter($parent);
|
||||
}
|
||||
}
|
||||
|
||||
private function targetServer(): Server
|
||||
{
|
||||
$target = $this->step->operation->target;
|
||||
|
||||
$server = match (true) {
|
||||
$target instanceof ServiceReplica => $target->server,
|
||||
$target instanceof Service => $target->replicas()->with('server')->first()?->server ?: $target->server,
|
||||
$target instanceof ServiceSlice => $target->service->replicas()->with('server')->first()?->server ?: $target->service->server,
|
||||
$target instanceof Environment => $target->services()->with(['server', 'replicas.server'])->get()
|
||||
->flatMap(fn (Service $service) => $service->replicas->pluck('server')->filter())
|
||||
->first() ?: $target->services()->with('server')->get()->pluck('server')->filter()->first(),
|
||||
default => null,
|
||||
};
|
||||
|
||||
if (! $server instanceof Server) {
|
||||
throw new \RuntimeException('Operation target does not have a server for SSH execution.');
|
||||
}
|
||||
|
||||
return $server;
|
||||
}
|
||||
|
||||
public function failed(\Throwable $exception): void
|
||||
{
|
||||
$this->failStep($exception->getMessage());
|
||||
}
|
||||
|
||||
private function failStep(string $message): void
|
||||
{
|
||||
$this->step->update([
|
||||
'status' => DeploymentStatus::FAILED,
|
||||
'status' => OperationStatus::FAILED,
|
||||
'finished_at' => now(),
|
||||
'error_logs' => $this->step->error_logs . "\n" . trim($exception->getMessage()),
|
||||
'error_logs' => $this->step->error_logs."\n".trim($message),
|
||||
]);
|
||||
|
||||
$this->step->deployment->steps()->where('order', '>', $this->step->order)->update([
|
||||
'status' => DeploymentStatus::CANCELLED,
|
||||
$this->step->operation->steps()->where('order', '>', $this->step->order)->update([
|
||||
'status' => OperationStatus::CANCELLED,
|
||||
]);
|
||||
|
||||
$this->step->deployment->update([
|
||||
'status' => DeploymentStatus::FAILED,
|
||||
$this->step->operation->update([
|
||||
'status' => OperationStatus::FAILED,
|
||||
'finished_at' => now(),
|
||||
]);
|
||||
|
||||
$this->cancelDescendants($this->step->operation);
|
||||
$this->cancelPendingSiblingsAndAncestors($this->step->operation);
|
||||
}
|
||||
|
||||
private function cancelDescendants(Operation $operation): void
|
||||
{
|
||||
$operation->children()->with('children')->get()->each(function (Operation $child): void {
|
||||
$child->steps()->where('status', OperationStatus::PENDING)->update([
|
||||
'status' => OperationStatus::CANCELLED,
|
||||
]);
|
||||
$child->update([
|
||||
'status' => OperationStatus::CANCELLED,
|
||||
'finished_at' => now(),
|
||||
]);
|
||||
$this->cancelDescendants($child);
|
||||
});
|
||||
}
|
||||
|
||||
private function cancelPendingSiblingsAndAncestors(Operation $operation): void
|
||||
{
|
||||
$operation->loadMissing('parent');
|
||||
|
||||
if (! $operation->parent) {
|
||||
return;
|
||||
}
|
||||
|
||||
$operation->parent->children()
|
||||
->where('id', '!=', $operation->id)
|
||||
->whereIn('status', [OperationStatus::PENDING, OperationStatus::IN_PROGRESS])
|
||||
->get()
|
||||
->each(function (Operation $sibling): void {
|
||||
$sibling->steps()->where('status', OperationStatus::PENDING)->update([
|
||||
'status' => OperationStatus::CANCELLED,
|
||||
]);
|
||||
$sibling->update([
|
||||
'status' => OperationStatus::CANCELLED,
|
||||
'finished_at' => now(),
|
||||
]);
|
||||
$this->cancelDescendants($sibling);
|
||||
});
|
||||
|
||||
$operation->parent->update([
|
||||
'status' => OperationStatus::FAILED,
|
||||
'finished_at' => now(),
|
||||
]);
|
||||
|
||||
$this->cancelPendingSiblingsAndAncestors($operation->parent);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user