harry/keystone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add managed registry provisioning, pruning, and readiness tracking

Browse Source
This commit is contained in:
Harry Bayliss
2026-06-08 20:44:16 +01:00
parent 5b977c1f41
commit 3a851db08f
52 changed files with 2706 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
app/Jobs/Environments/DeployEnvironment.php
View File

@@ -15,9 +15,13 @@ use App\Enums\ServiceEndpointScope;
use App\Models\Environment;
use App\Models\EnvironmentAttachment;
use App\Models\Operation;
use App\Models\Registry;
use App\Models\Server;
use App\Models\Service;
use App\Models\ServiceReplica;
use App\Services\Compose\ComposeRenderer;
use App\Services\Registries\RegistryDockerAuthScript;
use App\Services\Registries\RegistryResolver;
use App\Support\CaddyRouteRenderer;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Queue\Queueable;
@@ -67,6 +71,7 @@ class DeployEnvironment implements ShouldQueue
$artifact = app(PlanBuildArtifact::class)->execute($this->environment, $commitSha);
$artifact = app(BuildApplicationArtifact::class)->execute($artifact, $operation);
$registry = app(RegistryResolver::class)->buildRegistryFor($this->environment->application->organisation);
foreach ($services as $service) {
$service->update([
@@ -80,8 +85,8 @@ class DeployEnvironment implements ShouldQueue
'status' => OperationStatus::PENDING,
]);
$this->createServiceDeploySteps($child, $service, $commitSha, $artifact->image_digest);
$this->createReplicaDeployOperations($child, $service, $artifact->registry_ref);
$this->createServiceDeploySteps($child, $service, $commitSha, $artifact->image_digest, $artifact->registry_ref);
$this->createReplicaDeployOperations($child, $service, $artifact->registry_ref, $registry);
}
$this->createGatewayOperations($operation);
@@ -100,19 +105,20 @@ class DeployEnvironment implements ShouldQueue
->all();
}
private function createServiceDeploySteps(Operation $operation, Service $service, string $commitSha, string $imageDigest): void
private function createServiceDeploySteps(Operation $operation, Service $service, string $commitSha, string $imageDigest, ?string $imageReference = null): void
{
foreach ($this->serviceDeployScripts($service, $commitSha, $imageDigest) as $index => $step) {
foreach ($this->serviceDeployScripts($service, $commitSha, $imageDigest, $imageReference) as $index => $step) {
$operation->steps()->create([
'name' => $step['name'],
'order' => $index + 1,
'status' => OperationStatus::PENDING,
'script' => $step['script'],
'secrets' => $step['secrets'] ?? null,
]);
}
}
private function createReplicaDeployOperations(Operation $parent, Service $service, ?string $imageReference = null): void
private function createReplicaDeployOperations(Operation $parent, Service $service, ?string $imageReference = null, ?Registry $registry = null): void
{
$replicas = $this->ensureServiceReplicas($service);
@@ -133,12 +139,13 @@ class DeployEnvironment implements ShouldQueue
'health_status' => 'unknown',
]);
foreach ($this->replicaDeployScripts($service, $replica, $imageReference) as $index => $step) {
foreach ($this->replicaDeployScripts($service, $replica, $imageReference, $registry, $serviceReplica) as $index => $step) {
$operation->steps()->create([
'name' => $step['name'],
'order' => $index + 1,
'status' => OperationStatus::PENDING,
'script' => $step['script'],
'secrets' => $step['secrets'] ?? null,
]);
}
}
@@ -244,7 +251,7 @@ class DeployEnvironment implements ShouldQueue
/**
* @return array<int, array{name: string, script: string}>
*/
private function serviceDeployScripts(Service $service, string $commitSha, string $imageDigest): array
private function serviceDeployScripts(Service $service, string $commitSha, string $imageDigest, ?string $imageReference = null): array
{
$servicePath = $this->servicePath($service);
$composePath = "{$servicePath}/compose.yml";
@@ -264,7 +271,7 @@ class DeployEnvironment implements ShouldQueue
],
[
'name' => 'Render Compose files',
'script' => $this->composeUploadScript($service),
'script' => $this->composeUploadScript($service, $this->fullImageReference($imageReference, $imageDigest)),
],
];
@@ -311,17 +318,32 @@ class DeployEnvironment implements ShouldQueue
}
/**
* @return array<int, array{name: string, script: string}>
* @return array<int, array{name: string, script: string, secrets?: array<string, string>}>
*/
private function replicaDeployScripts(Service $service, int $replica, ?string $imageReference = null): array
private function replicaDeployScripts(Service $service, int $replica, ?string $imageReference = null, ?Registry $registry = null, ?ServiceReplica $serviceReplica = null): array
{
$composePath = $this->servicePath($service).'/compose.yml';
$project = "keystone_service_{$service->id}_replica_{$replica}";
$serviceKey = $this->serviceKey($service);
$targetServer = $serviceReplica?->server ?: $service->server;
$steps = [];
$steps = [
[
'name' => "Render replica {$replica} Compose files",
'script' => $this->composeUploadScript($service, $this->fullImageReference($imageReference, $service->available_image_digest)),
],
];
if ($imageReference && $service->available_image_digest) {
if ($registry instanceof Registry && $registry->credentials) {
$auth = app(RegistryDockerAuthScript::class)->forRuntime($registry, $this->dockerAuthUser($targetServer));
$steps[] = [
'name' => "Configure registry auth for replica {$replica}",
'script' => $auth['script'],
'secrets' => $auth['secrets'],
];
}
$steps[] = [
'name' => "Pull image for replica {$replica}",
'script' => 'docker pull '.escapeshellarg($imageReference.'@'.$service->available_image_digest),
@@ -354,16 +376,21 @@ class DeployEnvironment implements ShouldQueue
];
}
private function composeUploadScript(Service $service): string
private function dockerAuthUser(?Server $server): string
{
return 'root';
}
private function composeUploadScript(Service $service, ?string $fullImageReference = null): string
{
$servicePath = $this->servicePath($service);
try {
$renderer = app(ComposeRenderer::class);
$compose = $renderer->render($service);
$compose = $renderer->render($this->serviceForCompose($service, $fullImageReference));
$env = $renderer->renderEnvironmentFile($service);
} catch (InvalidArgumentException) {
$compose = "services:\n {$this->serviceKey($service)}:\n image: \"{$service->available_image_digest}\"\n";
$compose = "services:\n {$this->serviceKey($service)}:\n image: \"".($fullImageReference ?: $service->available_image_digest)."\"\n";
$env = '';
}
@@ -374,6 +401,27 @@ class DeployEnvironment implements ShouldQueue
]);
}
private function fullImageReference(?string $imageReference, ?string $imageDigest): ?string
{
if (! $imageReference || ! $imageDigest) {
return null;
}
return $imageReference.'@'.$imageDigest;
}
private function serviceForCompose(Service $service, ?string $fullImageReference): Service
{
if (! $fullImageReference) {
return $service;
}
$clone = clone $service;
$clone->available_image_digest = $fullImageReference;
return $clone;
}
/**
* @return array<int, array{name: string, script: string}>
*/

116
app/Jobs/Services/RunStep.php
View File

@@ -2,11 +2,15 @@
namespace App\Jobs\Services;
use App\Enums\BuildArtifactStatus;
use App\Enums\OperationStatus;
use App\Enums\RegistryType;
use App\Enums\ServiceStatus;
use App\Models\BuildArtifact;
use App\Models\Environment;
use App\Models\Operation;
use App\Models\OperationStep;
use App\Models\Registry;
use App\Models\Server;
use App\Models\Service;
use App\Models\ServiceReplica;
@@ -110,6 +114,9 @@ class RunStep implements ShouldQueue
if ($operation->is($this->step->operation)) {
$this->markTargetCompleted();
}
$this->markRegistryHealthOperationCompleted($operation);
$this->markRegistryMaintenanceOperationCompleted($operation);
}
private function dispatchNextChildOperation(Operation $operation): bool
@@ -166,6 +173,7 @@ class RunStep implements ShouldQueue
$target instanceof ServiceReplica => $target->server,
$target instanceof Service => $target->replicas()->with('server')->first()?->server ?: $target->server,
$target instanceof ServiceSlice => $target->service->replicas()->with('server')->first()?->server ?: $target->service->server,
$target instanceof Server => $target,
$target instanceof Environment => $target->services()->with(['server', 'replicas.server'])->get()
->flatMap(fn (Service $service) => $service->replicas->pluck('server')->filter())
->first() ?: $target->services()->with('server')->get()->pluck('server')->filter()->first(),
@@ -190,10 +198,12 @@ class RunStep implements ShouldQueue
'status' => OperationStatus::FAILED,
'finished_at' => now(),
'error_logs' => $this->step->error_logs."\n".trim($message),
'secrets' => null,
]);
$this->step->operation->steps()->where('order', '>', $this->step->order)->update([
'status' => OperationStatus::CANCELLED,
'secrets' => null,
]);
$this->step->operation->update([
@@ -203,13 +213,114 @@ class RunStep implements ShouldQueue
$this->cancelDescendants($this->step->operation);
$this->cancelPendingSiblingsAndAncestors($this->step->operation);
$this->markRegistryHealthOperationFailed($this->step->operation, trim($message));
}
private function markRegistryHealthOperationCompleted(Operation $operation): void
{
if ($operation->kind !== \App\Enums\OperationKind::REGISTRY_HEALTH_CHECK) {
return;
}
if ($operation->parent_id !== null) {
return;
}
$registry = $this->managedRegistryForOperation($operation);
if (! $registry instanceof Registry) {
return;
}
$checks = collect($registry->readiness_checks ?? [])
->map(fn (): string => 'passed')
->all();
$registry->forceFill([
'readiness_checks' => $checks,
])->save();
$registry->markHealthy('Managed registry smoke checks passed.');
}
private function markRegistryHealthOperationFailed(Operation $operation, string $message): void
{
if ($operation->kind !== \App\Enums\OperationKind::REGISTRY_HEALTH_CHECK) {
return;
}
$registry = $this->managedRegistryForOperation($operation);
$registry?->markUnhealthy('Managed registry smoke check failed: '.$message);
}
private function markRegistryMaintenanceOperationCompleted(Operation $operation): void
{
if ($operation->kind !== \App\Enums\OperationKind::REGISTRY_MAINTENANCE) {
return;
}
$registry = $this->managedRegistryForOperation($operation);
if (! $registry instanceof Registry) {
return;
}
$artifactIds = collect($operation->metadata['artifact_ids'] ?? [])
->filter(fn ($id): bool => is_numeric($id))
->map(fn ($id): int => (int) $id)
->values();
BuildArtifact::query()
->whereIn('id', $artifactIds)
->where('status', BuildArtifactStatus::PRUNABLE)
->where('registry_ref', 'like', rtrim((string) $registry->url, '/').'/%')
->each(function ($artifact): void {
$artifact->update([
'status' => BuildArtifactStatus::PRUNED,
'metadata' => [
...($artifact->metadata ?? []),
'pruned_at' => now()->toIso8601String(),
],
]);
});
}
private function managedRegistryForOperation(Operation $operation): ?Registry
{
$registryId = $operation->metadata['registry_id'] ?? $operation->parent?->metadata['registry_id'] ?? null;
if ($registryId) {
$registry = Registry::query()
->where('type', RegistryType::MANAGED->value)
->find($registryId);
if ($registry instanceof Registry) {
return $registry;
}
}
$server = $operation->target;
if (! $server instanceof Server) {
$server = $operation->parent?->target;
}
if (! $server instanceof Server) {
return null;
}
return Registry::query()
->where('type', RegistryType::MANAGED->value)
->where('control_server_id', $server->id)
->first();
}
private function cancelDescendants(Operation $operation): void
{
$operation->children()->with('children')->get()->each(function (Operation $child): void {
$child->steps()->where('status', OperationStatus::PENDING)->update([
$child->steps()->whereIn('status', [OperationStatus::PENDING, OperationStatus::IN_PROGRESS])->update([
'status' => OperationStatus::CANCELLED,
'secrets' => null,
]);
$child->update([
'status' => OperationStatus::CANCELLED,
@@ -232,8 +343,9 @@ class RunStep implements ShouldQueue
->whereIn('status', [OperationStatus::PENDING, OperationStatus::IN_PROGRESS])
->get()
->each(function (Operation $sibling): void {
$sibling->steps()->where('status', OperationStatus::PENDING)->update([
$sibling->steps()->whereIn('status', [OperationStatus::PENDING, OperationStatus::IN_PROGRESS])->update([
'status' => OperationStatus::CANCELLED,
'secrets' => null,
]);
$sibling->update([
'status' => OperationStatus::CANCELLED,