harry/keystone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add managed registry provisioning, pruning, and readiness tracking

Browse Source
This commit is contained in:
Harry Bayliss
2026-06-08 20:44:16 +01:00
parent 5b977c1f41
commit 3a851db08f
52 changed files with 2706 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
app/Actions/Environments/BuildApplicationArtifact.php
View File

@@ -4,11 +4,15 @@ namespace App\Actions\Environments;
use App\Enums\BuildArtifactStatus;
use App\Enums\BuildStrategy;
use App\Enums\RegistryType;
use App\Models\BuildArtifact;
use App\Models\Operation;
use App\Models\Registry;
use App\Models\Server;
use App\Models\Service;
use App\Services\Operations\RemoteCommandRunner;
use App\Services\Registries\RegistryDockerAuthScript;
use App\Services\Registries\RegistryResolver;
use RuntimeException;
class BuildApplicationArtifact
@@ -61,6 +65,18 @@ class BuildApplicationArtifact
private function buildServer(BuildArtifact $artifact): Server
{
$buildServerId = (int) ($artifact->metadata['build_server_id'] ?? 0);
if ($buildServerId > 0) {
$server = Server::find($buildServerId);
if ($server instanceof Server && $server->build_enabled) {
return $server;
}
throw new RuntimeException('Configured build server is missing or not build-enabled.');
}
if ($artifact->builtByService instanceof Service) {
$server = $artifact->builtByService->replicas->first()?->server ?: $artifact->builtByService->server;
@@ -70,7 +86,7 @@ class BuildApplicationArtifact
}
if (($artifact->metadata['build_strategy'] ?? null) === BuildStrategy::DEDICATED_BUILDER->value) {
throw new RuntimeException('Dedicated builder strategy requires a builder service.');
throw new RuntimeException('Dedicated builder strategy requires a builder service or build-enabled server.');
}
$services = $artifact->environment->services()
@@ -107,9 +123,13 @@ class BuildApplicationArtifact
$operationDirectory = '/home/keystone/operations/build-'.$artifact->id.'-'.str()->random(8);
$imageReference = $artifact->registry_ref ?: $artifact->image_tag;
$pushCommand = $strategy === BuildStrategy::DEDICATED_BUILDER && $artifact->registry_ref
? "\ndocker push ".escapeshellarg($imageReference)
: '';
$publishCommands = $artifact->registry_ref && $strategy !== BuildStrategy::EXTERNAL_REGISTRY
? [
...$this->pushDigestCommands($imageReference),
]
: [
'digest=$(docker image inspect --format '.escapeshellarg('{{if .RepoDigests}}{{index .RepoDigests 0}}{{else}}{{.Id}}{{end}}').' '.escapeshellarg($imageReference).')',
];
return implode("\n", [
'set -euo pipefail',
@@ -126,25 +146,91 @@ class BuildApplicationArtifact
'git clone --depth 1 --branch '.escapeshellarg($artifact->environment->branch).' '.escapeshellarg($application->repository_url).' "$source_dir"',
$this->writeFileCommand('$source_dir/Dockerfile.keystone', $this->dockerfile($artifact)),
'cd "$source_dir"',
'docker build --file Dockerfile.keystone --tag '.escapeshellarg($imageReference).' .'.$pushCommand,
'digest=$(docker image inspect --format '.escapeshellarg('{{if .RepoDigests}}{{index .RepoDigests 0}}{{else}}{{.Id}}{{end}}').' '.escapeshellarg($imageReference).')',
...$this->registryMaintenanceLockCommands($artifact),
...$this->buildAuthCommands($artifact),
'docker build --file Dockerfile.keystone --tag '.escapeshellarg($imageReference).' .',
...$publishCommands,
'printf "image_digest=%s\n" "$digest"',
]);
}
/**
* @return array<int, string>
*/
private function registryMaintenanceLockCommands(BuildArtifact $artifact): array
{
if (($artifact->metadata['registry_type'] ?? null) !== RegistryType::MANAGED->value) {
return [];
}
return [
'install -d -m 700 -o root -g root /home/keystone/registry',
'exec 9>/home/keystone/registry/maintenance.lock',
'flock 9',
];
}
/**
* @return array<int, string>
*/
private function buildAuthCommands(BuildArtifact $artifact): array
{
if (($artifact->metadata['registry_type'] ?? null) !== RegistryType::MANAGED->value) {
return [];
}
$registry = app(RegistryResolver::class)->buildRegistryFor($artifact->environment->application->organisation);
if (! $registry instanceof Registry || $registry->type !== RegistryType::MANAGED || ! $registry->credentials) {
throw new RuntimeException('Managed registry build credentials are not configured.');
}
$auth = app(RegistryDockerAuthScript::class)->forBuild($registry, 'root');
$script = $auth['script'];
foreach ($auth['secrets'] as $key => $value) {
$script = str_replace("[!{$key}!]", $value, $script);
}
return [$script];
}
private function manifestDigestScript(BuildArtifact $artifact): string
{
$imageReference = $artifact->registry_ref ?: $artifact->image_tag;
return implode("\n", [
'set -euo pipefail',
'manifest=$(docker manifest inspect '.escapeshellarg($imageReference).')',
'digest=$(printf "%s" "$manifest" | sed -n '.escapeshellarg('s/.*"digest": "\(sha256:[^"]*\)".*/\1/p').' | head -n 1)',
'test -n "$digest"',
...$this->manifestDigestCommands($imageReference),
'printf "image_digest=%s\n" "$digest"',
]);
}
/**
* @return array<int, string>
*/
private function manifestDigestCommands(string $imageReference): array
{
return [
'inspect_output=$(docker buildx imagetools inspect '.escapeshellarg($imageReference).')',
'digest=$(printf "%s\n" "$inspect_output" | sed -n '.escapeshellarg('s/^Digest:[[:space:]]*\(sha256:[^[:space:]]*\).*/\1/p').' | head -n 1)',
'test -n "$digest"',
];
}
/**
* @return array<int, string>
*/
private function pushDigestCommands(string $imageReference): array
{
return [
'push_output=$(docker push '.escapeshellarg($imageReference).')',
'printf "%s\n" "$push_output"',
'digest=$(printf "%s\n" "$push_output" | sed -n '.escapeshellarg('s/.*digest: \(sha256:[^[:space:]]*\).*/\1/p').' | tail -n 1)',
'test -n "$digest"',
];
}
private function dockerfile(BuildArtifact $artifact): string
{
$service = $artifact->environment->services()
@@ -176,7 +262,7 @@ DOCKERFILE;
private function digestFromOutput(string $output): string
{
if (preg_match('/image_digest=(?<digest>\S+)/', $output, $matches)) {
return $this->digestFromOutput($matches['digest']);
$output = $matches['digest'];
}
if (str_contains($output, '@')) {

47
app/Actions/Environments/PlanBuildArtifact.php
View File

@@ -4,13 +4,21 @@ namespace App\Actions\Environments;
use App\Enums\BuildArtifactStatus;
use App\Enums\BuildStrategy;
use App\Enums\RegistryType;
use App\Enums\ServiceCategory;
use App\Models\BuildArtifact;
use App\Models\Environment;
use App\Services\Registries\ImageReference;
use App\Services\Registries\RegistryResolver;
use RuntimeException;
class PlanBuildArtifact
{
public function __construct(
private readonly RegistryResolver $registryResolver,
private readonly ImageReference $imageReference,
) {}
public function execute(Environment $environment, string $commitSha): BuildArtifact
{
$environment->loadMissing(['application.organisation.registries', 'services.replicas']);
@@ -26,36 +34,48 @@ class PlanBuildArtifact
}
$targetServerCount = $this->targetServerCount($environment);
$registry = $environment->application->organisation->registries()->first();
$registry = $this->registryResolver->buildRegistryFor($environment->application->organisation);
$registryType = $this->registryType($registry);
if ($targetServerCount > 1 && ! $registry) {
throw new RuntimeException('A registry is required before building artifacts for multi-server deployments.');
$blocker = $this->registryResolver->managedRegistryBlockerFor($environment->application->organisation);
throw new RuntimeException($blocker ?: 'A registry is required before building artifacts for multi-server deployments.');
}
$builder = $environment->application->organisation->services()
->where('category', ServiceCategory::BUILDER)
->first();
$buildServerId = null;
if ($registryType === RegistryType::MANAGED) {
$buildServerId = (int) $registry->control_server_id;
if ($buildServerId <= 0) {
throw new RuntimeException('A control/build server is required for managed registry builds.');
}
}
$strategy = match (true) {
$registryType === RegistryType::MANAGED => BuildStrategy::DEDICATED_BUILDER,
$registry !== null => BuildStrategy::EXTERNAL_REGISTRY,
$builder !== null => BuildStrategy::DEDICATED_BUILDER,
default => BuildStrategy::TARGET_SERVER,
};
$imageTag = str($environment->application->name)
->slug()
->append(':'.substr($commitSha, 0, 12))
->value();
$imageTag = $this->imageReference->tagFor($environment, $commitSha, $registry);
return $environment->buildArtifacts()->create([
'commit_sha' => $commitSha,
'image_tag' => $imageTag,
'registry_ref' => $registry ? rtrim((string) $registry->url, '/').'/'.$imageTag : null,
'registry_ref' => $registry ? $this->imageReference->registryReference($registry, $imageTag) : null,
'built_by_service_id' => $builder?->id,
'status' => BuildArtifactStatus::PENDING,
'metadata' => [
'build_strategy' => $strategy->value,
'registry_type' => $registryType?->value,
'target_server_count' => $targetServerCount,
'build_server_id' => $buildServerId,
],
]);
}
@@ -73,4 +93,17 @@ class PlanBuildArtifact
return $environment->services->sum('desired_replicas') > 1 ? 2 : 1;
}
private function registryType(mixed $registry): ?RegistryType
{
if (! $registry) {
return null;
}
if ($registry->type instanceof RegistryType) {
return $registry->type;
}
return RegistryType::tryFrom((string) $registry->type);
}
}

27
app/Actions/Environments/PlanEnvironmentDeployment.php
View File

@@ -7,9 +7,14 @@ use App\Enums\DeployPolicy;
use App\Enums\EnvironmentAttachmentRole;
use App\Enums\SchedulerMode;
use App\Models\Environment;
use App\Services\Registries\RegistryResolver;
class PlanEnvironmentDeployment
{
public function __construct(
private readonly RegistryResolver $registryResolver,
) {}
public function execute(Environment $environment): EnvironmentDeploymentPlan
{
$environment->loadMissing([
@@ -40,9 +45,9 @@ class PlanEnvironmentDeployment
return new EnvironmentDeploymentPlan(
services: $deployableServices->all(),
dependencies: $dependencies->all(),
requiresRegistry: $targetServerCount > 1 && $environment->application->organisation->registries()->doesntExist(),
requiresRegistry: $targetServerCount > 1 && ! $this->registryResolver->buildRegistryFor($environment->application->organisation),
warnings: $this->warnings($environment),
blockers: $this->blockers($environment),
blockers: $this->blockers($environment, $targetServerCount),
);
}
@@ -74,18 +79,28 @@ class PlanEnvironmentDeployment
/**
* @return array<int, string>
*/
private function blockers(Environment $environment): array
private function blockers(Environment $environment, int $targetServerCount): array
{
$blockers = [];
if ($targetServerCount > 1 && ! $this->registryResolver->buildRegistryFor($environment->application->organisation)) {
$blocker = $this->registryResolver->managedRegistryBlockerFor($environment->application->organisation);
if ($blocker !== null) {
$blockers[] = $blocker;
}
}
if (! $environment->scheduler_enabled || $environment->scheduler_mode !== SchedulerMode::SINGLE) {
return [];
return $blockers;
}
$target = $environment->services->firstWhere('id', $environment->scheduler_target_service_id);
if ($target && $target->desired_replicas > 1 && in_array('scheduler', $target->process_roles ?? [], true)) {
return ['Scheduler mode single requires the scheduler target service to run exactly one replica.'];
$blockers[] = 'Scheduler mode single requires the scheduler target service to run exactly one replica.';
}
return [];
return $blockers;
}
}

79
app/Actions/Registries/CreateManagedRegistryMaintenanceOperation.php Normal file
View File

@@ -0,0 +1,79 @@
<?php
namespace App\Actions\Registries;
use App\Enums\BuildArtifactStatus;
use App\Enums\OperationKind;
use App\Enums\OperationStatus;
use App\Models\BuildArtifact;
use App\Models\Operation;
use App\Models\Registry;
use App\Models\Server;
use App\Services\Registries\ManagedRegistryOperationScripts;
use App\Services\Registries\ManagedRegistryRetention;
use RuntimeException;
class CreateManagedRegistryMaintenanceOperation
{
public function __construct(
private readonly ManagedRegistryRetention $retention,
private readonly ManagedRegistryOperationScripts $scripts,
) {}
public function execute(Registry $registry): Operation
{
$server = $registry->controlServer;
if (! $server instanceof Server) {
throw new RuntimeException('A control/build server is required to prune the managed registry.');
}
$activeBuilds = $registry->organisation->applications()
->whereHas('environments.buildArtifacts', fn ($query) => $query
->where('status', BuildArtifactStatus::BUILDING)
->where('registry_ref', 'like', rtrim((string) $registry->url, '/').'/%'))
->exists();
if ($activeBuilds) {
throw new RuntimeException('Managed registry pruning cannot run while builds are active.');
}
$this->retention->markPrunable($registry);
$artifacts = $this->prunableArtifacts($registry);
$maintenance = $this->scripts->maintenance($registry, $artifacts);
$operation = $server->operations()->create([
'kind' => OperationKind::REGISTRY_MAINTENANCE,
'status' => OperationStatus::PENDING,
'metadata' => [
'registry_id' => $registry->id,
'artifact_ids' => $artifacts->pluck('id')->values()->all(),
],
]);
$operation->steps()->create([
'name' => 'Delete prunable manifests and run registry GC',
'order' => 1,
'status' => OperationStatus::PENDING,
'script' => $maintenance['script'],
'secrets' => $maintenance['secrets'],
]);
return $operation->refresh();
}
/**
* @return \Illuminate\Support\Collection<int, BuildArtifact>
*/
private function prunableArtifacts(Registry $registry): \Illuminate\Support\Collection
{
return $registry->organisation->applications()
->with(['environments.buildArtifacts' => fn ($query) => $query
->where('status', BuildArtifactStatus::PRUNABLE)
->where('registry_ref', 'like', rtrim((string) $registry->url, '/').'/%')])
->get()
->flatMap(fn ($application) => $application->environments)
->flatMap(fn ($environment) => $environment->buildArtifacts)
->values();
}
}

44
app/Actions/Registries/CreateManagedRegistryProvisionOperation.php Normal file
View File

@@ -0,0 +1,44 @@
<?php
namespace App\Actions\Registries;
use App\Enums\OperationKind;
use App\Enums\OperationStatus;
use App\Models\Operation;
use App\Models\Registry;
use App\Models\Server;
use App\Services\Registries\ManagedRegistryOperationScripts;
use RuntimeException;
class CreateManagedRegistryProvisionOperation
{
public function __construct(
private readonly ManagedRegistryOperationScripts $scripts,
) {}
public function execute(Registry $registry): Operation
{
$server = $registry->controlServer;
if (! $server instanceof Server) {
throw new RuntimeException('A control/build server is required to provision the managed registry.');
}
$provision = $this->scripts->provision($registry);
$operation = $server->operations()->create([
'kind' => OperationKind::REGISTRY_PROVISION,
'status' => OperationStatus::PENDING,
]);
$operation->steps()->create([
'name' => 'Install managed Docker registry',
'order' => 1,
'status' => OperationStatus::PENDING,
'script' => $provision['script'],
'secrets' => $provision['secrets'],
]);
return $operation->refresh();
}
}

86
app/Actions/Registries/CreateManagedRegistrySmokeCheckOperation.php Normal file
View File

@@ -0,0 +1,86 @@
<?php
namespace App\Actions\Registries;
use App\Enums\OperationKind;
use App\Enums\OperationStatus;
use App\Models\Operation;
use App\Models\Registry;
use App\Models\Server;
use App\Services\Registries\ManagedRegistryOperationScripts;
use RuntimeException;
class CreateManagedRegistrySmokeCheckOperation
{
public function __construct(
private readonly ManagedRegistryOperationScripts $scripts,
) {}
/**
* @param iterable<int, Server> $runtimeServers
*/
public function execute(Registry $registry, ?Server $buildServer = null, iterable $runtimeServers = []): Operation
{
$controlServer = $registry->controlServer;
if (! $controlServer instanceof Server) {
throw new RuntimeException('A control/build server is required to check the managed registry.');
}
$buildServer ??= $controlServer;
$smokeRef = rtrim((string) $registry->url, '/').'/keystone/smoke/server-'.$buildServer->id.':latest';
$checks = [
'control_https' => 'pending',
'build_push' => 'pending',
];
foreach ($runtimeServers as $server) {
$checks['runtime_pull_server_'.$server->id] = 'pending';
}
$registry->forceFill([
'readiness_checks' => $checks,
'health_status' => 'pending',
'ready_at' => null,
])->save();
$operation = $controlServer->operations()->create([
'kind' => OperationKind::REGISTRY_HEALTH_CHECK,
'status' => OperationStatus::PENDING,
'metadata' => [
'registry_id' => $registry->id,
],
]);
$build = $this->scripts->smokeCheck($registry, $buildServer, 'build', $smokeRef);
$operation->steps()->create([
'name' => 'Check registry HTTPS and build push',
'order' => 1,
'status' => OperationStatus::PENDING,
'script' => $build['script'],
'secrets' => $build['secrets'],
]);
$order = 2;
foreach ($runtimeServers as $server) {
$runtime = $this->scripts->smokeCheck($registry, $server, 'runtime', $smokeRef);
$child = $server->operations()->create([
'kind' => OperationKind::REGISTRY_HEALTH_CHECK,
'parent_id' => $operation->id,
'status' => OperationStatus::PENDING,
'metadata' => [
'registry_id' => $registry->id,
],
]);
$child->steps()->create([
'name' => 'Check runtime registry pull on '.$server->name,
'order' => $order++,
'status' => OperationStatus::PENDING,
'script' => $runtime['script'],
'secrets' => $runtime['secrets'],
]);
}
return $operation->refresh();
}
}

42
app/Actions/Registries/CreateRegistryAuthOperation.php Normal file
View File

@@ -0,0 +1,42 @@
<?php
namespace App\Actions\Registries;
use App\Enums\OperationKind;
use App\Enums\OperationStatus;
use App\Models\Operation;
use App\Models\Registry;
use App\Models\Server;
use App\Services\Registries\RegistryDockerAuthScript;
use InvalidArgumentException;
class CreateRegistryAuthOperation
{
public function __construct(
private readonly RegistryDockerAuthScript $registryDockerAuthScript,
) {}
public function execute(Registry $registry, Server $server, string $scope): Operation
{
$auth = match ($scope) {
'build' => $this->registryDockerAuthScript->forBuild($registry, 'root'),
'runtime' => $this->registryDockerAuthScript->forRuntime($registry, 'root'),
default => throw new InvalidArgumentException('Registry auth scope must be build or runtime.'),
};
$operation = $server->operations()->create([
'kind' => OperationKind::CREDENTIAL_ROTATION,
'status' => OperationStatus::PENDING,
]);
$operation->steps()->create([
'name' => 'Configure '.$scope.' registry auth',
'order' => 1,
'status' => OperationStatus::PENDING,
'script' => $auth['script'],
'secrets' => $auth['secrets'],
]);
return $operation->refresh();
}
}