harry/keystone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add managed registry provisioning, pruning, and readiness tracking

Browse Source
This commit is contained in:
Harry Bayliss
2026-06-08 20:44:16 +01:00
parent 5b977c1f41
commit 3a851db08f
52 changed files with 2706 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
app/Actions/Environments/BuildApplicationArtifact.php
View File

@@ -4,11 +4,15 @@ namespace App\Actions\Environments;
use App\Enums\BuildArtifactStatus;
use App\Enums\BuildStrategy;
use App\Enums\RegistryType;
use App\Models\BuildArtifact;
use App\Models\Operation;
use App\Models\Registry;
use App\Models\Server;
use App\Models\Service;
use App\Services\Operations\RemoteCommandRunner;
use App\Services\Registries\RegistryDockerAuthScript;
use App\Services\Registries\RegistryResolver;
use RuntimeException;
class BuildApplicationArtifact
@@ -61,6 +65,18 @@ class BuildApplicationArtifact
private function buildServer(BuildArtifact $artifact): Server
{
$buildServerId = (int) ($artifact->metadata['build_server_id'] ?? 0);
if ($buildServerId > 0) {
$server = Server::find($buildServerId);
if ($server instanceof Server && $server->build_enabled) {
return $server;
}
throw new RuntimeException('Configured build server is missing or not build-enabled.');
}
if ($artifact->builtByService instanceof Service) {
$server = $artifact->builtByService->replicas->first()?->server ?: $artifact->builtByService->server;
@@ -70,7 +86,7 @@ class BuildApplicationArtifact
}
if (($artifact->metadata['build_strategy'] ?? null) === BuildStrategy::DEDICATED_BUILDER->value) {
throw new RuntimeException('Dedicated builder strategy requires a builder service.');
throw new RuntimeException('Dedicated builder strategy requires a builder service or build-enabled server.');
}
$services = $artifact->environment->services()
@@ -107,9 +123,13 @@ class BuildApplicationArtifact
$operationDirectory = '/home/keystone/operations/build-'.$artifact->id.'-'.str()->random(8);
$imageReference = $artifact->registry_ref ?: $artifact->image_tag;
$pushCommand = $strategy === BuildStrategy::DEDICATED_BUILDER && $artifact->registry_ref
? "\ndocker push ".escapeshellarg($imageReference)
: '';
$publishCommands = $artifact->registry_ref && $strategy !== BuildStrategy::EXTERNAL_REGISTRY
? [
...$this->pushDigestCommands($imageReference),
]
: [
'digest=$(docker image inspect --format '.escapeshellarg('{{if .RepoDigests}}{{index .RepoDigests 0}}{{else}}{{.Id}}{{end}}').' '.escapeshellarg($imageReference).')',
];
return implode("\n", [
'set -euo pipefail',
@@ -126,25 +146,91 @@ class BuildApplicationArtifact
'git clone --depth 1 --branch '.escapeshellarg($artifact->environment->branch).' '.escapeshellarg($application->repository_url).' "$source_dir"',
$this->writeFileCommand('$source_dir/Dockerfile.keystone', $this->dockerfile($artifact)),
'cd "$source_dir"',
'docker build --file Dockerfile.keystone --tag '.escapeshellarg($imageReference).' .'.$pushCommand,
'digest=$(docker image inspect --format '.escapeshellarg('{{if .RepoDigests}}{{index .RepoDigests 0}}{{else}}{{.Id}}{{end}}').' '.escapeshellarg($imageReference).')',
...$this->registryMaintenanceLockCommands($artifact),
...$this->buildAuthCommands($artifact),
'docker build --file Dockerfile.keystone --tag '.escapeshellarg($imageReference).' .',
...$publishCommands,
'printf "image_digest=%s\n" "$digest"',
]);
}
/**
* @return array<int, string>
*/
private function registryMaintenanceLockCommands(BuildArtifact $artifact): array
{
if (($artifact->metadata['registry_type'] ?? null) !== RegistryType::MANAGED->value) {
return [];
}
return [
'install -d -m 700 -o root -g root /home/keystone/registry',
'exec 9>/home/keystone/registry/maintenance.lock',
'flock 9',
];
}
/**
* @return array<int, string>
*/
private function buildAuthCommands(BuildArtifact $artifact): array
{
if (($artifact->metadata['registry_type'] ?? null) !== RegistryType::MANAGED->value) {
return [];
}
$registry = app(RegistryResolver::class)->buildRegistryFor($artifact->environment->application->organisation);
if (! $registry instanceof Registry || $registry->type !== RegistryType::MANAGED || ! $registry->credentials) {
throw new RuntimeException('Managed registry build credentials are not configured.');
}
$auth = app(RegistryDockerAuthScript::class)->forBuild($registry, 'root');
$script = $auth['script'];
foreach ($auth['secrets'] as $key => $value) {
$script = str_replace("[!{$key}!]", $value, $script);
}
return [$script];
}
private function manifestDigestScript(BuildArtifact $artifact): string
{
$imageReference = $artifact->registry_ref ?: $artifact->image_tag;
return implode("\n", [
'set -euo pipefail',
'manifest=$(docker manifest inspect '.escapeshellarg($imageReference).')',
'digest=$(printf "%s" "$manifest" | sed -n '.escapeshellarg('s/.*"digest": "\(sha256:[^"]*\)".*/\1/p').' | head -n 1)',
'test -n "$digest"',
...$this->manifestDigestCommands($imageReference),
'printf "image_digest=%s\n" "$digest"',
]);
}
/**
* @return array<int, string>
*/
private function manifestDigestCommands(string $imageReference): array
{
return [
'inspect_output=$(docker buildx imagetools inspect '.escapeshellarg($imageReference).')',
'digest=$(printf "%s\n" "$inspect_output" | sed -n '.escapeshellarg('s/^Digest:[[:space:]]*\(sha256:[^[:space:]]*\).*/\1/p').' | head -n 1)',
'test -n "$digest"',
];
}
/**
* @return array<int, string>
*/
private function pushDigestCommands(string $imageReference): array
{
return [
'push_output=$(docker push '.escapeshellarg($imageReference).')',
'printf "%s\n" "$push_output"',
'digest=$(printf "%s\n" "$push_output" | sed -n '.escapeshellarg('s/.*digest: \(sha256:[^[:space:]]*\).*/\1/p').' | tail -n 1)',
'test -n "$digest"',
];
}
private function dockerfile(BuildArtifact $artifact): string
{
$service = $artifact->environment->services()
@@ -176,7 +262,7 @@ DOCKERFILE;
private function digestFromOutput(string $output): string
{
if (preg_match('/image_digest=(?<digest>\S+)/', $output, $matches)) {
return $this->digestFromOutput($matches['digest']);
$output = $matches['digest'];
}
if (str_contains($output, '@')) {

47
app/Actions/Environments/PlanBuildArtifact.php
View File

@@ -4,13 +4,21 @@ namespace App\Actions\Environments;
use App\Enums\BuildArtifactStatus;
use App\Enums\BuildStrategy;
use App\Enums\RegistryType;
use App\Enums\ServiceCategory;
use App\Models\BuildArtifact;
use App\Models\Environment;
use App\Services\Registries\ImageReference;
use App\Services\Registries\RegistryResolver;
use RuntimeException;
class PlanBuildArtifact
{
public function __construct(
private readonly RegistryResolver $registryResolver,
private readonly ImageReference $imageReference,
) {}
public function execute(Environment $environment, string $commitSha): BuildArtifact
{
$environment->loadMissing(['application.organisation.registries', 'services.replicas']);
@@ -26,36 +34,48 @@ class PlanBuildArtifact
}
$targetServerCount = $this->targetServerCount($environment);
$registry = $environment->application->organisation->registries()->first();
$registry = $this->registryResolver->buildRegistryFor($environment->application->organisation);
$registryType = $this->registryType($registry);
if ($targetServerCount > 1 && ! $registry) {
throw new RuntimeException('A registry is required before building artifacts for multi-server deployments.');
$blocker = $this->registryResolver->managedRegistryBlockerFor($environment->application->organisation);
throw new RuntimeException($blocker ?: 'A registry is required before building artifacts for multi-server deployments.');
}
$builder = $environment->application->organisation->services()
->where('category', ServiceCategory::BUILDER)
->first();
$buildServerId = null;
if ($registryType === RegistryType::MANAGED) {
$buildServerId = (int) $registry->control_server_id;
if ($buildServerId <= 0) {
throw new RuntimeException('A control/build server is required for managed registry builds.');
}
}
$strategy = match (true) {
$registryType === RegistryType::MANAGED => BuildStrategy::DEDICATED_BUILDER,
$registry !== null => BuildStrategy::EXTERNAL_REGISTRY,
$builder !== null => BuildStrategy::DEDICATED_BUILDER,
default => BuildStrategy::TARGET_SERVER,
};
$imageTag = str($environment->application->name)
->slug()
->append(':'.substr($commitSha, 0, 12))
->value();
$imageTag = $this->imageReference->tagFor($environment, $commitSha, $registry);
return $environment->buildArtifacts()->create([
'commit_sha' => $commitSha,
'image_tag' => $imageTag,
'registry_ref' => $registry ? rtrim((string) $registry->url, '/').'/'.$imageTag : null,
'registry_ref' => $registry ? $this->imageReference->registryReference($registry, $imageTag) : null,
'built_by_service_id' => $builder?->id,
'status' => BuildArtifactStatus::PENDING,
'metadata' => [
'build_strategy' => $strategy->value,
'registry_type' => $registryType?->value,
'target_server_count' => $targetServerCount,
'build_server_id' => $buildServerId,
],
]);
}
@@ -73,4 +93,17 @@ class PlanBuildArtifact
return $environment->services->sum('desired_replicas') > 1 ? 2 : 1;
}
private function registryType(mixed $registry): ?RegistryType
{
if (! $registry) {
return null;
}
if ($registry->type instanceof RegistryType) {
return $registry->type;
}
return RegistryType::tryFrom((string) $registry->type);
}
}

27
app/Actions/Environments/PlanEnvironmentDeployment.php
View File

@@ -7,9 +7,14 @@ use App\Enums\DeployPolicy;
use App\Enums\EnvironmentAttachmentRole;
use App\Enums\SchedulerMode;
use App\Models\Environment;
use App\Services\Registries\RegistryResolver;
class PlanEnvironmentDeployment
{
public function __construct(
private readonly RegistryResolver $registryResolver,
) {}
public function execute(Environment $environment): EnvironmentDeploymentPlan
{
$environment->loadMissing([
@@ -40,9 +45,9 @@ class PlanEnvironmentDeployment
return new EnvironmentDeploymentPlan(
services: $deployableServices->all(),
dependencies: $dependencies->all(),
requiresRegistry: $targetServerCount > 1 && $environment->application->organisation->registries()->doesntExist(),
requiresRegistry: $targetServerCount > 1 && ! $this->registryResolver->buildRegistryFor($environment->application->organisation),
warnings: $this->warnings($environment),
blockers: $this->blockers($environment),
blockers: $this->blockers($environment, $targetServerCount),
);
}
@@ -74,18 +79,28 @@ class PlanEnvironmentDeployment
/**
* @return array<int, string>
*/
private function blockers(Environment $environment): array
private function blockers(Environment $environment, int $targetServerCount): array
{
$blockers = [];
if ($targetServerCount > 1 && ! $this->registryResolver->buildRegistryFor($environment->application->organisation)) {
$blocker = $this->registryResolver->managedRegistryBlockerFor($environment->application->organisation);
if ($blocker !== null) {
$blockers[] = $blocker;
}
}
if (! $environment->scheduler_enabled || $environment->scheduler_mode !== SchedulerMode::SINGLE) {
return [];
return $blockers;
}
$target = $environment->services->firstWhere('id', $environment->scheduler_target_service_id);
if ($target && $target->desired_replicas > 1 && in_array('scheduler', $target->process_roles ?? [], true)) {
return ['Scheduler mode single requires the scheduler target service to run exactly one replica.'];
$blockers[] = 'Scheduler mode single requires the scheduler target service to run exactly one replica.';
}
return [];
return $blockers;
}
}

79
app/Actions/Registries/CreateManagedRegistryMaintenanceOperation.php Normal file
View File

@@ -0,0 +1,79 @@
<?php
namespace App\Actions\Registries;
use App\Enums\BuildArtifactStatus;
use App\Enums\OperationKind;
use App\Enums\OperationStatus;
use App\Models\BuildArtifact;
use App\Models\Operation;
use App\Models\Registry;
use App\Models\Server;
use App\Services\Registries\ManagedRegistryOperationScripts;
use App\Services\Registries\ManagedRegistryRetention;
use RuntimeException;
class CreateManagedRegistryMaintenanceOperation
{
public function __construct(
private readonly ManagedRegistryRetention $retention,
private readonly ManagedRegistryOperationScripts $scripts,
) {}
public function execute(Registry $registry): Operation
{
$server = $registry->controlServer;
if (! $server instanceof Server) {
throw new RuntimeException('A control/build server is required to prune the managed registry.');
}
$activeBuilds = $registry->organisation->applications()
->whereHas('environments.buildArtifacts', fn ($query) => $query
->where('status', BuildArtifactStatus::BUILDING)
->where('registry_ref', 'like', rtrim((string) $registry->url, '/').'/%'))
->exists();
if ($activeBuilds) {
throw new RuntimeException('Managed registry pruning cannot run while builds are active.');
}
$this->retention->markPrunable($registry);
$artifacts = $this->prunableArtifacts($registry);
$maintenance = $this->scripts->maintenance($registry, $artifacts);
$operation = $server->operations()->create([
'kind' => OperationKind::REGISTRY_MAINTENANCE,
'status' => OperationStatus::PENDING,
'metadata' => [
'registry_id' => $registry->id,
'artifact_ids' => $artifacts->pluck('id')->values()->all(),
],
]);
$operation->steps()->create([
'name' => 'Delete prunable manifests and run registry GC',
'order' => 1,
'status' => OperationStatus::PENDING,
'script' => $maintenance['script'],
'secrets' => $maintenance['secrets'],
]);
return $operation->refresh();
}
/**
* @return \Illuminate\Support\Collection<int, BuildArtifact>
*/
private function prunableArtifacts(Registry $registry): \Illuminate\Support\Collection
{
return $registry->organisation->applications()
->with(['environments.buildArtifacts' => fn ($query) => $query
->where('status', BuildArtifactStatus::PRUNABLE)
->where('registry_ref', 'like', rtrim((string) $registry->url, '/').'/%')])
->get()
->flatMap(fn ($application) => $application->environments)
->flatMap(fn ($environment) => $environment->buildArtifacts)
->values();
}
}

44
app/Actions/Registries/CreateManagedRegistryProvisionOperation.php Normal file
View File

@@ -0,0 +1,44 @@
<?php
namespace App\Actions\Registries;
use App\Enums\OperationKind;
use App\Enums\OperationStatus;
use App\Models\Operation;
use App\Models\Registry;
use App\Models\Server;
use App\Services\Registries\ManagedRegistryOperationScripts;
use RuntimeException;
class CreateManagedRegistryProvisionOperation
{
public function __construct(
private readonly ManagedRegistryOperationScripts $scripts,
) {}
public function execute(Registry $registry): Operation
{
$server = $registry->controlServer;
if (! $server instanceof Server) {
throw new RuntimeException('A control/build server is required to provision the managed registry.');
}
$provision = $this->scripts->provision($registry);
$operation = $server->operations()->create([
'kind' => OperationKind::REGISTRY_PROVISION,
'status' => OperationStatus::PENDING,
]);
$operation->steps()->create([
'name' => 'Install managed Docker registry',
'order' => 1,
'status' => OperationStatus::PENDING,
'script' => $provision['script'],
'secrets' => $provision['secrets'],
]);
return $operation->refresh();
}
}

86
app/Actions/Registries/CreateManagedRegistrySmokeCheckOperation.php Normal file
View File

@@ -0,0 +1,86 @@
<?php
namespace App\Actions\Registries;
use App\Enums\OperationKind;
use App\Enums\OperationStatus;
use App\Models\Operation;
use App\Models\Registry;
use App\Models\Server;
use App\Services\Registries\ManagedRegistryOperationScripts;
use RuntimeException;
class CreateManagedRegistrySmokeCheckOperation
{
public function __construct(
private readonly ManagedRegistryOperationScripts $scripts,
) {}
/**
* @param iterable<int, Server> $runtimeServers
*/
public function execute(Registry $registry, ?Server $buildServer = null, iterable $runtimeServers = []): Operation
{
$controlServer = $registry->controlServer;
if (! $controlServer instanceof Server) {
throw new RuntimeException('A control/build server is required to check the managed registry.');
}
$buildServer ??= $controlServer;
$smokeRef = rtrim((string) $registry->url, '/').'/keystone/smoke/server-'.$buildServer->id.':latest';
$checks = [
'control_https' => 'pending',
'build_push' => 'pending',
];
foreach ($runtimeServers as $server) {
$checks['runtime_pull_server_'.$server->id] = 'pending';
}
$registry->forceFill([
'readiness_checks' => $checks,
'health_status' => 'pending',
'ready_at' => null,
])->save();
$operation = $controlServer->operations()->create([
'kind' => OperationKind::REGISTRY_HEALTH_CHECK,
'status' => OperationStatus::PENDING,
'metadata' => [
'registry_id' => $registry->id,
],
]);
$build = $this->scripts->smokeCheck($registry, $buildServer, 'build', $smokeRef);
$operation->steps()->create([
'name' => 'Check registry HTTPS and build push',
'order' => 1,
'status' => OperationStatus::PENDING,
'script' => $build['script'],
'secrets' => $build['secrets'],
]);
$order = 2;
foreach ($runtimeServers as $server) {
$runtime = $this->scripts->smokeCheck($registry, $server, 'runtime', $smokeRef);
$child = $server->operations()->create([
'kind' => OperationKind::REGISTRY_HEALTH_CHECK,
'parent_id' => $operation->id,
'status' => OperationStatus::PENDING,
'metadata' => [
'registry_id' => $registry->id,
],
]);
$child->steps()->create([
'name' => 'Check runtime registry pull on '.$server->name,
'order' => $order++,
'status' => OperationStatus::PENDING,
'script' => $runtime['script'],
'secrets' => $runtime['secrets'],
]);
}
return $operation->refresh();
}
}

42
app/Actions/Registries/CreateRegistryAuthOperation.php Normal file
View File

@@ -0,0 +1,42 @@
<?php
namespace App\Actions\Registries;
use App\Enums\OperationKind;
use App\Enums\OperationStatus;
use App\Models\Operation;
use App\Models\Registry;
use App\Models\Server;
use App\Services\Registries\RegistryDockerAuthScript;
use InvalidArgumentException;
class CreateRegistryAuthOperation
{
public function __construct(
private readonly RegistryDockerAuthScript $registryDockerAuthScript,
) {}
public function execute(Registry $registry, Server $server, string $scope): Operation
{
$auth = match ($scope) {
'build' => $this->registryDockerAuthScript->forBuild($registry, 'root'),
'runtime' => $this->registryDockerAuthScript->forRuntime($registry, 'root'),
default => throw new InvalidArgumentException('Registry auth scope must be build or runtime.'),
};
$operation = $server->operations()->create([
'kind' => OperationKind::CREDENTIAL_ROTATION,
'status' => OperationStatus::PENDING,
]);
$operation->steps()->create([
'name' => 'Configure '.$scope.' registry auth',
'order' => 1,
'status' => OperationStatus::PENDING,
'script' => $auth['script'],
'secrets' => $auth['secrets'],
]);
return $operation->refresh();
}
}

49
app/Console/Commands/CheckManagedRegistry.php Normal file
View File

@@ -0,0 +1,49 @@
<?php
namespace App\Console\Commands;
use App\Actions\Registries\CreateManagedRegistrySmokeCheckOperation;
use App\Enums\RegistryType;
use App\Models\Registry;
use App\Models\Server;
use Illuminate\Console\Command;
class CheckManagedRegistry extends Command
{
protected $signature = 'keystone:managed-registry:check
{registry : Managed registry id}
{--build-server= : Build server id, defaults to the registry control server}
{--runtime-server=* : Runtime server id to pull the smoke image}
{--dispatch : Dispatch the first operation step immediately}';
protected $description = 'Create managed registry HTTPS/auth/push/pull smoke-check operations.';
public function handle(CreateManagedRegistrySmokeCheckOperation $operations): int
{
$registry = Registry::query()
->where('type', RegistryType::MANAGED->value)
->findOrFail((int) $this->argument('registry'));
$buildServer = $this->option('build-server')
? Server::query()
->where('organisation_id', $registry->organisation_id)
->findOrFail((int) $this->option('build-server'))
: null;
$runtimeServers = collect($this->option('runtime-server'))
->map(fn (string $serverId): Server => Server::query()
->where('organisation_id', $registry->organisation_id)
->findOrFail((int) $serverId));
$operation = $operations->execute($registry, $buildServer, $runtimeServers);
$this->info("Created registry smoke-check operation {$operation->id}.");
if ($this->option('dispatch')) {
$operation->steps()->orderBy('order')->first()?->dispatchJob();
$this->info('Dispatched registry smoke-check operation.');
}
return self::SUCCESS;
}
}

87
app/Console/Commands/ProvisionManagedRegistry.php Normal file
View File

@@ -0,0 +1,87 @@
<?php
namespace App\Console\Commands;
use App\Actions\Registries\CreateManagedRegistryProvisionOperation;
use App\Models\Organisation;
use App\Models\Server;
use App\Services\Registries\ManagedRegistryHealth;
use App\Services\Registries\ManagedRegistryProvisioner;
use Illuminate\Console\Command;
class ProvisionManagedRegistry extends Command
{
protected $signature = 'keystone:managed-registry:provision
{organisation : Organisation id or slug}
{--url= : HTTPS registry hostname}
{--control-server= : Control/build server id}
{--storage-path= : Registry storage path}
{--retention= : Successful artifacts to retain per environment}
{--create-operation : Create the remote registry install/proxy operation}
{--dispatch : Dispatch the first operation step immediately}
{--mark-healthy : Mark the persisted registry ready after configuration validation}';
protected $description = 'Persist and optionally install a first-party managed Docker registry.';
public function handle(ManagedRegistryProvisioner $provisioner, ManagedRegistryHealth $health, CreateManagedRegistryProvisionOperation $operations): int
{
$organisationKey = (string) $this->argument('organisation');
$organisation = Organisation::query()
->where('id', $organisationKey)
->orWhere('slug', $organisationKey)
->firstOrFail();
$url = (string) ($this->option('url') ?: config('keystone.managed_registry.url'));
if ($url === '') {
$this->error('Provide --url or KEYSTONE_MANAGED_REGISTRY_URL.');
return self::FAILURE;
}
$controlServer = $this->option('control-server')
? Server::query()
->where('organisation_id', $organisation->id)
->findOrFail((int) $this->option('control-server'))
: null;
$registry = $provisioner->provision(
organisation: $organisation,
url: $url,
controlServer: $controlServer,
storagePath: $this->option('storage-path') ? (string) $this->option('storage-path') : null,
retention: $this->option('retention') ? (int) $this->option('retention') : null,
);
$blocker = $health->readinessBlocker($registry);
if ($this->option('mark-healthy') && $blocker !== null && $blocker !== 'Managed registry has not passed readiness checks.') {
$this->error($blocker);
return self::FAILURE;
}
if ($this->option('mark-healthy')) {
$registry->markHealthy('Marked ready by provisioning command.');
$blocker = null;
}
$this->info("Managed registry {$registry->url} persisted for {$organisation->name}.");
if ($blocker !== null) {
$this->warn($blocker);
}
if ($this->option('create-operation')) {
$operation = $operations->execute($registry);
$this->info("Created registry provision operation {$operation->id}.");
if ($this->option('dispatch')) {
$operation->steps()->orderBy('order')->first()?->dispatchJob();
$this->info('Dispatched registry provision operation.');
}
}
return self::SUCCESS;
}
}

40
app/Console/Commands/PruneManagedRegistry.php Normal file
View File

@@ -0,0 +1,40 @@
<?php
namespace App\Console\Commands;
use App\Actions\Registries\CreateManagedRegistryMaintenanceOperation;
use App\Enums\RegistryType;
use App\Models\Registry;
use Illuminate\Console\Command;
class PruneManagedRegistry extends Command
{
protected $signature = 'keystone:managed-registry:prune
{registry? : Managed registry id}
{--dispatch : Dispatch the first operation step immediately}';
protected $description = 'Create managed registry manifest deletion and garbage-collection operations.';
public function handle(CreateManagedRegistryMaintenanceOperation $operations): int
{
$registries = Registry::query()
->where('type', RegistryType::MANAGED->value)
->when($this->argument('registry'), fn ($query) => $query->whereKey((int) $this->argument('registry')))
->get();
$count = 0;
foreach ($registries as $registry) {
$operation = $operations->execute($registry);
$count++;
if ($this->option('dispatch')) {
$operation->steps()->orderBy('order')->first()?->dispatchJob();
}
}
$this->info("Created {$count} managed registry maintenance operation(s).");
return self::SUCCESS;
}
}

2
app/Enums/BuildArtifactStatus.php
View File

@@ -12,4 +12,6 @@ enum BuildArtifactStatus: string
case BUILDING = 'building';
case AVAILABLE = 'available';
case FAILED = 'failed';
case PRUNABLE = 'prunable';
case PRUNED = 'pruned';
}

3
app/Enums/OperationKind.php
View File

@@ -17,4 +17,7 @@ enum OperationKind: string
case GATEWAY_CUTOVER = 'gateway_cutover';
case CONFIG_CHANGE = 'config_change';
case CREDENTIAL_ROTATION = 'credential_rotation';
case REGISTRY_PROVISION = 'registry_provision';
case REGISTRY_HEALTH_CHECK = 'registry_health_check';
case REGISTRY_MAINTENANCE = 'registry_maintenance';
}

1
app/Enums/RegistryType.php
View File

@@ -8,6 +8,7 @@ enum RegistryType: string
{
use Arrayable;
case MANAGED = 'managed';
case GENERIC = 'generic';
case GITEA = 'gitea';
case GHCR = 'ghcr';

7
app/Http/Controllers/EnvironmentDeploymentController.php
View File

@@ -7,11 +7,16 @@ use App\Jobs\Environments\DeployEnvironment;
use App\Models\Application;
use App\Models\Environment;
use App\Models\Organisation;
use App\Services\Registries\RegistryResolver;
use Illuminate\Http\RedirectResponse;
use Illuminate\Support\Collection;
class EnvironmentDeploymentController extends Controller
{
public function __construct(
private readonly RegistryResolver $registryResolver,
) {}
public function store(StoreEnvironmentDeploymentRequest $request, Organisation $organisation, Application $application, Environment $environment): RedirectResponse
{
abort_unless(
@@ -22,7 +27,7 @@ class EnvironmentDeploymentController extends Controller
$environment->loadMissing('services.replicas');
if ($organisation->registries()->doesntExist() && $this->serverIdsFor($environment)->count() > 1) {
if (! $this->registryResolver->buildRegistryFor($organisation) && $this->serverIdsFor($environment)->count() > 1) {
return back()->with('error', 'Configure a registry before deploying this environment to multiple servers.');
}

10
app/Http/Controllers/OperationController.php
View File

@@ -122,6 +122,7 @@ class OperationController extends Controller
'status' => OperationStatus::CANCELLED,
'finished_at' => now(),
]);
$this->clearOperationSecrets($operation);
return redirect()
->route('operations.show', [
@@ -168,4 +169,13 @@ class OperationController extends Controller
default => false,
};
}
private function clearOperationSecrets(Operation $operation): void
{
$operation->steps()->update(['secrets' => null]);
$operation->children()->get()->each(function (Operation $child): void {
$this->clearOperationSecrets($child);
});
}
}

19
app/Http/Controllers/RegistryController.php
View File

@@ -30,7 +30,7 @@ class RegistryController extends Controller
Organisation::findOrFail($request->route('organisation'));
return inertia('registries/Create', [
'registryTypes' => array_values(RegistryType::toArray()),
'registryTypes' => $this->userConfigurableRegistryTypes(),
]);
}
@@ -81,7 +81,7 @@ class RegistryController extends Controller
return inertia('registries/Edit', [
'registry' => $registry,
'registryTypes' => array_values(RegistryType::toArray()),
'registryTypes' => $this->userConfigurableRegistryTypes(),
]);
}
@@ -91,6 +91,8 @@ class RegistryController extends Controller
/** @var Registry $registry */
$registry = $organisation->registries()->findOrFail($request->route('registry'));
abort_if($registry->type === RegistryType::MANAGED, 403);
$credentials = $registry->credentials ?? [];
$username = $request->string('username')->toString();
@@ -117,10 +119,23 @@ class RegistryController extends Controller
$organisation = Organisation::findOrFail($request->route('organisation'));
$registry = $organisation->registries()->findOrFail($request->route('registry'));
abort_if($registry->type === RegistryType::MANAGED, 403);
$registry->delete();
return redirect()
->route('organisations.show', ['organisation' => $organisation->id])
->with('success', 'Registry deleted.');
}
/**
* @return array<int, string>
*/
private function userConfigurableRegistryTypes(): array
{
return collect(RegistryType::toArray())
->reject(fn (string $type) => $type === RegistryType::MANAGED->value)
->values()
->all();
}
}

4
app/Http/Requests/StoreRegistryRequest.php
View File

@@ -25,8 +25,8 @@ class StoreRegistryRequest extends FormRequest
{
return [
'name' => ['required', 'string', 'max:255'],
'type' => ['required', Rule::enum(RegistryType::class)],
'url' => ['required', 'string', 'max:255'],
'type' => ['required', Rule::enum(RegistryType::class), Rule::notIn([RegistryType::MANAGED->value])],
'url' => ['required', 'string', 'max:255', 'not_regex:#^https?://#i'],
'username' => ['nullable', 'string', 'max:255'],
'password' => ['nullable', 'string', 'max:255'],
];

4
app/Http/Requests/UpdateRegistryRequest.php
View File

@@ -25,8 +25,8 @@ class UpdateRegistryRequest extends FormRequest
{
return [
'name' => ['required', 'string', 'max:255'],
'type' => ['required', Rule::enum(RegistryType::class)],
'url' => ['required', 'string', 'max:255'],
'type' => ['required', Rule::enum(RegistryType::class), Rule::notIn([RegistryType::MANAGED->value])],
'url' => ['required', 'string', 'max:255', 'not_regex:#^https?://#i'],
'username' => ['nullable', 'string', 'max:255'],
'password' => ['nullable', 'string', 'max:255'],
];

76
app/Jobs/Environments/DeployEnvironment.php
View File

@@ -15,9 +15,13 @@ use App\Enums\ServiceEndpointScope;
use App\Models\Environment;
use App\Models\EnvironmentAttachment;
use App\Models\Operation;
use App\Models\Registry;
use App\Models\Server;
use App\Models\Service;
use App\Models\ServiceReplica;
use App\Services\Compose\ComposeRenderer;
use App\Services\Registries\RegistryDockerAuthScript;
use App\Services\Registries\RegistryResolver;
use App\Support\CaddyRouteRenderer;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Queue\Queueable;
@@ -67,6 +71,7 @@ class DeployEnvironment implements ShouldQueue
$artifact = app(PlanBuildArtifact::class)->execute($this->environment, $commitSha);
$artifact = app(BuildApplicationArtifact::class)->execute($artifact, $operation);
$registry = app(RegistryResolver::class)->buildRegistryFor($this->environment->application->organisation);
foreach ($services as $service) {
$service->update([
@@ -80,8 +85,8 @@ class DeployEnvironment implements ShouldQueue
'status' => OperationStatus::PENDING,
]);
$this->createServiceDeploySteps($child, $service, $commitSha, $artifact->image_digest);
$this->createReplicaDeployOperations($child, $service, $artifact->registry_ref);
$this->createServiceDeploySteps($child, $service, $commitSha, $artifact->image_digest, $artifact->registry_ref);
$this->createReplicaDeployOperations($child, $service, $artifact->registry_ref, $registry);
}
$this->createGatewayOperations($operation);
@@ -100,19 +105,20 @@ class DeployEnvironment implements ShouldQueue
->all();
}
private function createServiceDeploySteps(Operation $operation, Service $service, string $commitSha, string $imageDigest): void
private function createServiceDeploySteps(Operation $operation, Service $service, string $commitSha, string $imageDigest, ?string $imageReference = null): void
{
foreach ($this->serviceDeployScripts($service, $commitSha, $imageDigest) as $index => $step) {
foreach ($this->serviceDeployScripts($service, $commitSha, $imageDigest, $imageReference) as $index => $step) {
$operation->steps()->create([
'name' => $step['name'],
'order' => $index + 1,
'status' => OperationStatus::PENDING,
'script' => $step['script'],
'secrets' => $step['secrets'] ?? null,
]);
}
}
private function createReplicaDeployOperations(Operation $parent, Service $service, ?string $imageReference = null): void
private function createReplicaDeployOperations(Operation $parent, Service $service, ?string $imageReference = null, ?Registry $registry = null): void
{
$replicas = $this->ensureServiceReplicas($service);
@@ -133,12 +139,13 @@ class DeployEnvironment implements ShouldQueue
'health_status' => 'unknown',
]);
foreach ($this->replicaDeployScripts($service, $replica, $imageReference) as $index => $step) {
foreach ($this->replicaDeployScripts($service, $replica, $imageReference, $registry, $serviceReplica) as $index => $step) {
$operation->steps()->create([
'name' => $step['name'],
'order' => $index + 1,
'status' => OperationStatus::PENDING,
'script' => $step['script'],
'secrets' => $step['secrets'] ?? null,
]);
}
}
@@ -244,7 +251,7 @@ class DeployEnvironment implements ShouldQueue
/**
* @return array<int, array{name: string, script: string}>
*/
private function serviceDeployScripts(Service $service, string $commitSha, string $imageDigest): array
private function serviceDeployScripts(Service $service, string $commitSha, string $imageDigest, ?string $imageReference = null): array
{
$servicePath = $this->servicePath($service);
$composePath = "{$servicePath}/compose.yml";
@@ -264,7 +271,7 @@ class DeployEnvironment implements ShouldQueue
],
[
'name' => 'Render Compose files',
'script' => $this->composeUploadScript($service),
'script' => $this->composeUploadScript($service, $this->fullImageReference($imageReference, $imageDigest)),
],
];
@@ -311,17 +318,32 @@ class DeployEnvironment implements ShouldQueue
}
/**
* @return array<int, array{name: string, script: string}>
* @return array<int, array{name: string, script: string, secrets?: array<string, string>}>
*/
private function replicaDeployScripts(Service $service, int $replica, ?string $imageReference = null): array
private function replicaDeployScripts(Service $service, int $replica, ?string $imageReference = null, ?Registry $registry = null, ?ServiceReplica $serviceReplica = null): array
{
$composePath = $this->servicePath($service).'/compose.yml';
$project = "keystone_service_{$service->id}_replica_{$replica}";
$serviceKey = $this->serviceKey($service);
$targetServer = $serviceReplica?->server ?: $service->server;
$steps = [];
$steps = [
[
'name' => "Render replica {$replica} Compose files",
'script' => $this->composeUploadScript($service, $this->fullImageReference($imageReference, $service->available_image_digest)),
],
];
if ($imageReference && $service->available_image_digest) {
if ($registry instanceof Registry && $registry->credentials) {
$auth = app(RegistryDockerAuthScript::class)->forRuntime($registry, $this->dockerAuthUser($targetServer));
$steps[] = [
'name' => "Configure registry auth for replica {$replica}",
'script' => $auth['script'],
'secrets' => $auth['secrets'],
];
}
$steps[] = [
'name' => "Pull image for replica {$replica}",
'script' => 'docker pull '.escapeshellarg($imageReference.'@'.$service->available_image_digest),
@@ -354,16 +376,21 @@ class DeployEnvironment implements ShouldQueue
];
}
private function composeUploadScript(Service $service): string
private function dockerAuthUser(?Server $server): string
{
return 'root';
}
private function composeUploadScript(Service $service, ?string $fullImageReference = null): string
{
$servicePath = $this->servicePath($service);
try {
$renderer = app(ComposeRenderer::class);
$compose = $renderer->render($service);
$compose = $renderer->render($this->serviceForCompose($service, $fullImageReference));
$env = $renderer->renderEnvironmentFile($service);
} catch (InvalidArgumentException) {
$compose = "services:\n {$this->serviceKey($service)}:\n image: \"{$service->available_image_digest}\"\n";
$compose = "services:\n {$this->serviceKey($service)}:\n image: \"".($fullImageReference ?: $service->available_image_digest)."\"\n";
$env = '';
}
@@ -374,6 +401,27 @@ class DeployEnvironment implements ShouldQueue
]);
}
private function fullImageReference(?string $imageReference, ?string $imageDigest): ?string
{
if (! $imageReference || ! $imageDigest) {
return null;
}
return $imageReference.'@'.$imageDigest;
}
private function serviceForCompose(Service $service, ?string $fullImageReference): Service
{
if (! $fullImageReference) {
return $service;
}
$clone = clone $service;
$clone->available_image_digest = $fullImageReference;
return $clone;
}
/**
* @return array<int, array{name: string, script: string}>
*/

116
app/Jobs/Services/RunStep.php
View File

@@ -2,11 +2,15 @@
namespace App\Jobs\Services;
use App\Enums\BuildArtifactStatus;
use App\Enums\OperationStatus;
use App\Enums\RegistryType;
use App\Enums\ServiceStatus;
use App\Models\BuildArtifact;
use App\Models\Environment;
use App\Models\Operation;
use App\Models\OperationStep;
use App\Models\Registry;
use App\Models\Server;
use App\Models\Service;
use App\Models\ServiceReplica;
@@ -110,6 +114,9 @@ class RunStep implements ShouldQueue
if ($operation->is($this->step->operation)) {
$this->markTargetCompleted();
}
$this->markRegistryHealthOperationCompleted($operation);
$this->markRegistryMaintenanceOperationCompleted($operation);
}
private function dispatchNextChildOperation(Operation $operation): bool
@@ -166,6 +173,7 @@ class RunStep implements ShouldQueue
$target instanceof ServiceReplica => $target->server,
$target instanceof Service => $target->replicas()->with('server')->first()?->server ?: $target->server,
$target instanceof ServiceSlice => $target->service->replicas()->with('server')->first()?->server ?: $target->service->server,
$target instanceof Server => $target,
$target instanceof Environment => $target->services()->with(['server', 'replicas.server'])->get()
->flatMap(fn (Service $service) => $service->replicas->pluck('server')->filter())
->first() ?: $target->services()->with('server')->get()->pluck('server')->filter()->first(),
@@ -190,10 +198,12 @@ class RunStep implements ShouldQueue
'status' => OperationStatus::FAILED,
'finished_at' => now(),
'error_logs' => $this->step->error_logs."\n".trim($message),
'secrets' => null,
]);
$this->step->operation->steps()->where('order', '>', $this->step->order)->update([
'status' => OperationStatus::CANCELLED,
'secrets' => null,
]);
$this->step->operation->update([
@@ -203,13 +213,114 @@ class RunStep implements ShouldQueue
$this->cancelDescendants($this->step->operation);
$this->cancelPendingSiblingsAndAncestors($this->step->operation);
$this->markRegistryHealthOperationFailed($this->step->operation, trim($message));
}
private function markRegistryHealthOperationCompleted(Operation $operation): void
{
if ($operation->kind !== \App\Enums\OperationKind::REGISTRY_HEALTH_CHECK) {
return;
}
if ($operation->parent_id !== null) {
return;
}
$registry = $this->managedRegistryForOperation($operation);
if (! $registry instanceof Registry) {
return;
}
$checks = collect($registry->readiness_checks ?? [])
->map(fn (): string => 'passed')
->all();
$registry->forceFill([
'readiness_checks' => $checks,
])->save();
$registry->markHealthy('Managed registry smoke checks passed.');
}
private function markRegistryHealthOperationFailed(Operation $operation, string $message): void
{
if ($operation->kind !== \App\Enums\OperationKind::REGISTRY_HEALTH_CHECK) {
return;
}
$registry = $this->managedRegistryForOperation($operation);
$registry?->markUnhealthy('Managed registry smoke check failed: '.$message);
}
private function markRegistryMaintenanceOperationCompleted(Operation $operation): void
{
if ($operation->kind !== \App\Enums\OperationKind::REGISTRY_MAINTENANCE) {
return;
}
$registry = $this->managedRegistryForOperation($operation);
if (! $registry instanceof Registry) {
return;
}
$artifactIds = collect($operation->metadata['artifact_ids'] ?? [])
->filter(fn ($id): bool => is_numeric($id))
->map(fn ($id): int => (int) $id)
->values();
BuildArtifact::query()
->whereIn('id', $artifactIds)
->where('status', BuildArtifactStatus::PRUNABLE)
->where('registry_ref', 'like', rtrim((string) $registry->url, '/').'/%')
->each(function ($artifact): void {
$artifact->update([
'status' => BuildArtifactStatus::PRUNED,
'metadata' => [
...($artifact->metadata ?? []),
'pruned_at' => now()->toIso8601String(),
],
]);
});
}
private function managedRegistryForOperation(Operation $operation): ?Registry
{
$registryId = $operation->metadata['registry_id'] ?? $operation->parent?->metadata['registry_id'] ?? null;
if ($registryId) {
$registry = Registry::query()
->where('type', RegistryType::MANAGED->value)
->find($registryId);
if ($registry instanceof Registry) {
return $registry;
}
}
$server = $operation->target;
if (! $server instanceof Server) {
$server = $operation->parent?->target;
}
if (! $server instanceof Server) {
return null;
}
return Registry::query()
->where('type', RegistryType::MANAGED->value)
->where('control_server_id', $server->id)
->first();
}
private function cancelDescendants(Operation $operation): void
{
$operation->children()->with('children')->get()->each(function (Operation $child): void {
$child->steps()->where('status', OperationStatus::PENDING)->update([
$child->steps()->whereIn('status', [OperationStatus::PENDING, OperationStatus::IN_PROGRESS])->update([
'status' => OperationStatus::CANCELLED,
'secrets' => null,
]);
$child->update([
'status' => OperationStatus::CANCELLED,
@@ -232,8 +343,9 @@ class RunStep implements ShouldQueue
->whereIn('status', [OperationStatus::PENDING, OperationStatus::IN_PROGRESS])
->get()
->each(function (Operation $sibling): void {
$sibling->steps()->where('status', OperationStatus::PENDING)->update([
$sibling->steps()->whereIn('status', [OperationStatus::PENDING, OperationStatus::IN_PROGRESS])->update([
'status' => OperationStatus::CANCELLED,
'secrets' => null,
]);
$sibling->update([
'status' => OperationStatus::CANCELLED,

8
app/Models/Application.php
View File

@@ -8,6 +8,7 @@ use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\Relations\MorphMany;
use Illuminate\Support\Str;
class Application extends Model
{
@@ -15,6 +16,13 @@ class Application extends Model
protected $guarded = [];
protected static function booted(): void
{
static::creating(function (Application $application): void {
$application->uuid ??= (string) Str::uuid();
});
}
protected function casts(): array
{
return [

8
app/Models/Environment.php
View File

@@ -8,6 +8,7 @@ use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\Relations\MorphMany;
use Illuminate\Support\Str;
class Environment extends Model
{
@@ -15,6 +16,13 @@ class Environment extends Model
protected $guarded = [];
protected static function booted(): void
{
static::creating(function (Environment $environment): void {
$environment->uuid ??= (string) Str::uuid();
});
}
protected function casts(): array
{
return [

1
app/Models/Operation.php
View File

@@ -30,6 +30,7 @@ class Operation extends Model
return [
'kind' => OperationKind::class,
'status' => OperationStatus::class,
'metadata' => 'array',
'started_at' => 'datetime',
'finished_at' => 'datetime',
];

4
app/Models/OperationStep.php
View File

@@ -13,6 +13,10 @@ class OperationStep extends Model
{
protected $guarded = [];
protected $hidden = [
'secrets',
];
protected $appends = [
'logs_excerpt',
'error_logs_excerpt',

34
app/Models/Registry.php
View File

@@ -5,6 +5,7 @@ namespace App\Models;
use App\Enums\RegistryType;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
use Illuminate\Support\Carbon;
class Registry extends Model
{
@@ -17,6 +18,9 @@ class Registry extends Model
return [
'type' => RegistryType::class,
'credentials' => 'encrypted:array',
'readiness_checks' => 'array',
'health_checked_at' => 'datetime',
'ready_at' => 'datetime',
];
}
@@ -24,4 +28,34 @@ class Registry extends Model
{
return $this->belongsTo(Organisation::class);
}
public function controlServer(): BelongsTo
{
return $this->belongsTo(Server::class, 'control_server_id');
}
public function markHealthy(?string $message = null): void
{
$this->forceFill([
'health_status' => 'healthy',
'health_message' => $message,
'health_checked_at' => Carbon::now(),
'ready_at' => $this->ready_at ?? Carbon::now(),
])->save();
}
public function markUnhealthy(string $message): void
{
$this->forceFill([
'health_status' => 'unhealthy',
'health_message' => $message,
'health_checked_at' => Carbon::now(),
'ready_at' => null,
])->save();
}
public function isReady(): bool
{
return $this->type !== RegistryType::MANAGED || ($this->ready_at !== null && $this->health_status === 'healthy');
}
}

2
app/Models/Server.php
View File

@@ -22,6 +22,8 @@ class Server extends Model
{
return [
'status' => ServerStatus::class,
'is_control_node' => 'boolean',
'build_enabled' => 'boolean',
];
}

54
app/Services/Registries/ImageReference.php Normal file
View File

@@ -0,0 +1,54 @@
<?php
namespace App\Services\Registries;
use App\Enums\RegistryType;
use App\Models\Environment;
use App\Models\Registry;
class ImageReference
{
public function tagFor(Environment $environment, string $commitSha, ?Registry $registry = null): string
{
$tag = substr($commitSha, 0, 12);
if ($this->registryType($registry) === RegistryType::MANAGED) {
$namespace = trim((string) config('keystone.managed_registry.namespace', 'keystone'), '/');
$applicationId = $environment->application->uuid ?: 'app-'.$environment->application_id;
$environmentId = $environment->uuid ?: 'env-'.$environment->id;
$path = $namespace.'/'.$applicationId.'/'.$environmentId;
return $path.':'.$tag;
}
return str($environment->application->name)
->slug()
->append(':'.$tag)
->value();
}
public function registryReference(Registry $registry, string $imageTag): string
{
return rtrim($this->registryHost((string) $registry->url), '/').'/'.ltrim($imageTag, '/');
}
private function registryHost(string $url): string
{
$host = preg_replace('#^https?://#', '', trim($url));
return $host === null ? trim($url) : $host;
}
private function registryType(?Registry $registry): ?RegistryType
{
if (! $registry instanceof Registry) {
return null;
}
if ($registry->type instanceof RegistryType) {
return $registry->type;
}
return RegistryType::tryFrom((string) $registry->type);
}
}

128
app/Services/Registries/ManagedRegistryHealth.php Normal file
View File

@@ -0,0 +1,128 @@
<?php
namespace App\Services\Registries;
use App\Enums\RegistryType;
use App\Models\Registry;
use App\Models\Server;
use Illuminate\Support\Facades\Http;
class ManagedRegistryHealth
{
public function check(Registry $registry): bool
{
if ($registry->type !== RegistryType::MANAGED) {
return true;
}
$message = $this->configurationBlocker($registry);
if ($message !== null) {
$registry->markUnhealthy($message);
return false;
}
try {
$response = Http::timeout(5)->get('https://'.$registry->url.'/v2/');
} catch (\Throwable $exception) {
$registry->markUnhealthy('Registry URL is not reachable over HTTPS: '.$exception->getMessage());
return false;
}
if (! in_array($response->status(), [200, 401], true)) {
$registry->markUnhealthy('Registry HTTPS check returned HTTP '.$response->status().'.');
return false;
}
$checks = $registry->readiness_checks ?? [];
$checks['control_https'] = 'passed';
$registry->forceFill([
'readiness_checks' => $checks,
])->save();
if ($this->readinessChecksPassed($registry->refresh())) {
$registry->markHealthy('Registry HTTPS endpoint and smoke checks passed.');
return true;
}
$registry->markUnhealthy('Registry HTTPS endpoint is reachable, but smoke checks have not all passed.');
return false;
}
public function readinessBlocker(Registry $registry): ?string
{
if ($registry->type !== RegistryType::MANAGED) {
return null;
}
$message = $this->configurationBlocker($registry);
if ($message !== null) {
return $message;
}
if ($registry->ready_at === null || $registry->health_status !== 'healthy') {
return 'Managed registry has not passed readiness checks.';
}
if (! $this->readinessChecksPassed($registry)) {
return 'Managed registry smoke checks have not all passed.';
}
return null;
}
private function configurationBlocker(Registry $registry): ?string
{
if (! $registry->url) {
return 'Managed registry URL is not configured.';
}
if (! str_contains((string) $registry->url, '.')) {
return 'Managed registry must use a resolvable HTTPS hostname.';
}
$credentials = $registry->credentials ?? [];
foreach (['build_username', 'build_password', 'runtime_username', 'runtime_password'] as $key) {
if (blank($credentials[$key] ?? null)) {
return 'Managed registry credentials are incomplete.';
}
}
$controlServer = $registry->controlServer;
if (! $controlServer instanceof Server) {
return 'A control/build server is required for managed registry builds.';
}
if (! $controlServer->build_enabled) {
return 'The managed registry control server is not build-enabled.';
}
return null;
}
private function readinessChecksPassed(Registry $registry): bool
{
$checks = $registry->readiness_checks ?? [];
if ($checks === []) {
return false;
}
foreach (['control_https', 'build_push'] as $requiredCheck) {
if (! array_key_exists($requiredCheck, $checks)) {
return false;
}
}
return collect($checks)->every(fn (mixed $status): bool => $status === 'passed');
}
}

189
app/Services/Registries/ManagedRegistryOperationScripts.php Normal file
View File

@@ -0,0 +1,189 @@
<?php
namespace App\Services\Registries;
use App\Models\BuildArtifact;
use App\Models\Registry;
use App\Models\Server;
class ManagedRegistryOperationScripts
{
/**
* @return array{script: string, secrets: array<string, string>}
*/
public function provision(Registry $registry): array
{
$credentials = $registry->credentials ?? [];
$host = $this->host($registry);
$storagePath = $registry->storage_path ?: (string) config('keystone.managed_registry.storage_path');
return [
'script' => implode("\n", [
'set -euo pipefail',
'storage_path='.escapeshellarg($storagePath),
'registry_host='.escapeshellarg($host),
'install -d -m 700 -o root -g root /home/keystone/registry/auth /home/keystone/registry/config',
'install -d -m 755 -o root -g root "$storage_path"',
'tmp_htpasswd=$(mktemp)',
'cleanup() { rm -f "$tmp_htpasswd"; unset build_password runtime_password; }',
'trap cleanup EXIT',
'build_password=$(printf %s '.escapeshellarg('[!build_password_base64!]').' | base64 -d)',
'runtime_password=$(printf %s '.escapeshellarg('[!runtime_password_base64!]').' | base64 -d)',
'printf %s "$build_password" | docker run -i --rm --entrypoint htpasswd httpd:2.4-alpine -Bni '.escapeshellarg((string) ($credentials['build_username'] ?? 'keystone-build')).' > "$tmp_htpasswd"',
'printf %s "$runtime_password" | docker run -i --rm --entrypoint htpasswd httpd:2.4-alpine -Bni '.escapeshellarg((string) ($credentials['runtime_username'] ?? 'keystone-runtime')).' >> "$tmp_htpasswd"',
'install -m 600 -o root -g root "$tmp_htpasswd" /home/keystone/registry/auth/htpasswd',
'cat > /home/keystone/registry/config/config.yml <<\'KEYSTONE_REGISTRY_CONFIG\'',
'version: 0.1',
'log:',
' fields:',
' service: registry',
'storage:',
' filesystem:',
' rootdirectory: /var/lib/registry',
' delete:',
' enabled: true',
'http:',
' addr: :5000',
'auth:',
' htpasswd:',
' realm: keystone-managed-registry',
' path: /auth/htpasswd',
'KEYSTONE_REGISTRY_CONFIG',
'docker rm -f keystone-managed-registry >/dev/null 2>&1 || true',
'docker run -d --name keystone-managed-registry --restart unless-stopped -p 127.0.0.1:5000:5000 -e REGISTRY_STORAGE_DELETE_ENABLED=true -v "$storage_path:/var/lib/registry" -v /home/keystone/registry/auth:/auth:ro -v /home/keystone/registry/config/config.yml:/etc/docker/registry/config.yml:ro registry:2',
'install -d -m 755 /home/keystone/gateway/Caddyfile.d',
'cat > /home/keystone/gateway/Caddyfile.d/managed-registry.caddy <<KEYSTONE_CADDY_REGISTRY',
'$registry_host {',
' reverse_proxy 127.0.0.1:5000',
'}',
'KEYSTONE_CADDY_REGISTRY',
'if test -d /home/keystone/gateway/Caddyfile.d; then cat /home/keystone/gateway/Caddyfile.d/*.caddy > /home/keystone/gateway/Caddyfile; fi',
'if docker ps --format \'{{.Names}}\' | grep -qx gateway-1; then docker exec gateway-1 caddy reload --config /etc/caddy/Caddyfile; fi',
'if docker ps --format \'{{.Names}}\' | grep -qx caddy; then docker exec caddy caddy reload --config /etc/caddy/Caddyfile; fi',
'if docker ps --format \'{{.Names}}\' | grep -Eqx \'(gateway-1|caddy)\'; then curl --fail --silent --show-error --location --head https://"$registry_host"/v2/ || test "$?" = "22"; else echo "Registry proxy reload skipped because no Caddy container is running."; fi',
]),
'secrets' => [
'build_password_base64' => base64_encode((string) ($credentials['build_password'] ?? '')),
'runtime_password_base64' => base64_encode((string) ($credentials['runtime_password'] ?? '')),
],
];
}
/**
* @return array{script: string, secrets: array<string, string>}
*/
public function smokeCheck(Registry $registry, Server $server, string $scope, ?string $imageReference = null): array
{
$credentials = $registry->credentials ?? [];
$username = (string) ($credentials[$scope.'_username'] ?? '');
$password = (string) ($credentials[$scope.'_password'] ?? '');
$host = $this->host($registry);
$repository = $imageReference ?: $host.'/keystone/smoke/server-'.$server->id.':latest';
$commands = [
'set -euo pipefail',
'registry_host='.escapeshellarg($host),
'image_ref='.escapeshellarg($repository),
'username='.escapeshellarg($username),
'password=$(printf %s '.escapeshellarg('[!registry_password_base64!]').' | base64 -d)',
'curl --fail --silent --show-error --user "$username:$password" https://"$registry_host"/v2/ >/dev/null',
'printf %s '.escapeshellarg('[!registry_password_base64!]').' | base64 -d | docker login "$registry_host" --username '.escapeshellarg($username).' --password-stdin >/dev/null',
];
if ($scope === 'build') {
$commands = [
...$commands,
'docker pull busybox:latest >/dev/null',
'docker tag busybox:latest "$image_ref"',
'docker push "$image_ref" >/dev/null',
'docker buildx imagetools inspect "$image_ref" >/dev/null',
'printf "smoke_ref=%s\n" "$image_ref"',
'unset password',
];
} else {
$commands = [
...$commands,
'docker pull "$image_ref" >/dev/null',
'unset password',
];
}
return [
'script' => implode("\n", $commands),
'secrets' => [
'registry_password_base64' => base64_encode($password),
],
];
}
/**
* @param iterable<int, BuildArtifact> $artifacts
* @return array{script: string, secrets: array<string, string>}
*/
public function maintenance(Registry $registry, iterable $artifacts): array
{
$credentials = $registry->credentials ?? [];
$host = $this->host($registry);
$deletions = [];
foreach ($artifacts as $artifact) {
$repository = $this->repositoryPath((string) $artifact->registry_ref);
if ($repository === '' || blank($artifact->image_digest)) {
continue;
}
$deletions[] = 'delete_manifest '.escapeshellarg($repository).' '.escapeshellarg((string) $artifact->image_digest).' || delete_failures=1';
}
return [
'script' => implode("\n", [
'set -euo pipefail',
'registry_host='.escapeshellarg($host),
'lock_file=/home/keystone/registry/maintenance.lock',
'exec 9>"$lock_file"',
'flock -n 9',
'username='.escapeshellarg((string) ($credentials['build_username'] ?? 'keystone-build')),
'password=$(printf %s '.escapeshellarg('[!registry_password_base64!]').' | base64 -d)',
'curl_config=$(mktemp)',
'registry_was_stopped=0',
'cleanup() { rm -f "$curl_config"; unset password auth_header; if test "$registry_was_stopped" = "1"; then docker start keystone-managed-registry >/dev/null 2>&1 || true; fi; }',
'trap cleanup EXIT',
'auth_header=$(printf "%s:%s" "$username" "$password" | base64 | tr -d "\n")',
'printf "header = \"Authorization: Basic %s\"\n" "$auth_header" > "$curl_config"',
'chmod 600 "$curl_config"',
'delete_failures=0',
'delete_manifest() {',
' repository="$1"',
' digest="$2"',
' status=$(curl --silent --show-error --output /tmp/keystone-registry-delete-response --write-out "%{http_code}" --request DELETE --config "$curl_config" --header "Accept: application/vnd.docker.distribution.manifest.v2+json" "https://$registry_host/v2/$repository/manifests/$digest" || true)',
' case "$status" in 2*|404) printf "deleted_manifest=%s@%s status=%s\n" "$repository" "$digest" "$status" ;; *) cat /tmp/keystone-registry-delete-response >&2; printf "delete_failed=%s@%s status=%s\n" "$repository" "$digest" "$status" >&2; return 1 ;; esac',
'}',
...$deletions,
'test "$delete_failures" = "0"',
'docker stop keystone-managed-registry',
'registry_was_stopped=1',
'docker run --rm -v /home/keystone/registry/config/config.yml:/etc/docker/registry/config.yml:ro -v '.escapeshellarg(($registry->storage_path ?: (string) config('keystone.managed_registry.storage_path')).':/var/lib/registry').' registry:2 garbage-collect --delete-untagged /etc/docker/registry/config.yml',
'docker start keystone-managed-registry',
'registry_was_stopped=0',
'unset password',
]),
'secrets' => [
'registry_password_base64' => base64_encode((string) ($credentials['build_password'] ?? '')),
],
];
}
private function host(Registry $registry): string
{
return rtrim((string) preg_replace('#^https?://#', '', (string) $registry->url), '/');
}
private function repositoryPath(string $reference): string
{
$withoutHost = str($reference)->after('/')->value();
$withoutTag = preg_replace('/:[^:\/]+$/', '', $withoutHost);
return trim((string) $withoutTag, '/');
}
}

64
app/Services/Registries/ManagedRegistryProvisioner.php Normal file
View File

@@ -0,0 +1,64 @@
<?php
namespace App\Services\Registries;
use App\Enums\RegistryType;
use App\Models\Organisation;
use App\Models\Registry;
use App\Models\Server;
use Illuminate\Support\Str;
class ManagedRegistryProvisioner
{
public function provision(Organisation $organisation, string $url, ?Server $controlServer = null, ?string $storagePath = null, ?int $retention = null): Registry
{
$registry = $organisation->registries()->firstOrNew([
'type' => RegistryType::MANAGED->value,
]);
$registry->fill([
'name' => 'Managed Registry',
'url' => $this->registryHost($url),
'storage_path' => $storagePath ?: (string) config('keystone.managed_registry.storage_path'),
'retention_successful_artifacts' => $retention ?: (int) config('keystone.managed_registry.retention.successful_artifacts_per_environment', 3),
'control_server_id' => $controlServer?->id,
'credentials' => $this->credentials($registry->credentials ?? []),
]);
if ($registry->health_status === null) {
$registry->health_status = 'pending';
}
$registry->save();
if ($controlServer instanceof Server) {
$controlServer->forceFill([
'is_control_node' => true,
'build_enabled' => true,
])->save();
}
return $registry->refresh();
}
/**
* @param array<string, mixed> $existing
* @return array<string, string>
*/
private function credentials(array $existing): array
{
return [
'build_username' => (string) ($existing['build_username'] ?? 'keystone-build'),
'build_password' => (string) ($existing['build_password'] ?? Str::password(40)),
'runtime_username' => (string) ($existing['runtime_username'] ?? 'keystone-runtime'),
'runtime_password' => (string) ($existing['runtime_password'] ?? Str::password(40)),
];
}
private function registryHost(string $url): string
{
$host = preg_replace('#^https?://#', '', trim($url));
return rtrim($host === null ? trim($url) : $host, '/');
}
}

80
app/Services/Registries/ManagedRegistryRetention.php Normal file
View File

@@ -0,0 +1,80 @@
<?php
namespace App\Services\Registries;
use App\Enums\BuildArtifactStatus;
use App\Enums\RegistryType;
use App\Models\BuildArtifact;
use App\Models\Environment;
use App\Models\Registry;
use Illuminate\Support\Collection;
class ManagedRegistryRetention
{
/**
* @return Collection<int, BuildArtifact>
*/
public function markPrunable(Registry $registry): Collection
{
if ($registry->type !== RegistryType::MANAGED) {
return collect();
}
$keep = max(1, (int) $registry->retention_successful_artifacts);
$updated = collect();
Environment::query()
->whereHas('buildArtifacts', fn ($query) => $query
->where('status', BuildArtifactStatus::AVAILABLE)
->where('registry_ref', 'like', rtrim((string) $registry->url, '/').'/%'))
->with(['services', 'buildArtifacts' => fn ($query) => $query
->where('status', BuildArtifactStatus::AVAILABLE)
->where('registry_ref', 'like', rtrim((string) $registry->url, '/').'/%')
->latest()])
->each(function (Environment $environment) use ($keep, $updated): void {
$activeDigests = $environment->services
->flatMap(fn ($service): array => [
$service->available_image_digest,
$service->current_image_digest,
])
->filter()
->all();
$environment->buildArtifacts
->skip($keep)
->filter(fn (BuildArtifact $artifact): bool => ! in_array($artifact->image_digest, $activeDigests, true))
->each(function (BuildArtifact $artifact) use ($updated): void {
$metadata = $artifact->metadata ?? [];
$artifact->update([
'status' => BuildArtifactStatus::PRUNABLE,
'metadata' => [
...$metadata,
'prunable_at' => now()->toIso8601String(),
'prune_command' => $this->deleteManifestCommand($artifact),
],
]);
$updated->push($artifact->refresh());
});
});
return $updated;
}
public function deleteManifestCommand(BuildArtifact $artifact): string
{
$reference = (string) ($artifact->registry_ref ?? '');
$digest = (string) $artifact->image_digest;
return 'curl --fail --silent --show-error --request DELETE '.escapeshellarg('https://'.$this->manifestPath($reference, $digest));
}
private function manifestPath(string $reference, string $digest): string
{
$hostAndPath = preg_replace('/:[^:\/]+$/', '', $reference);
$path = str($hostAndPath)->after('/')->value();
return $path === ''
? 'v2/'
: str($reference)->before('/')->value().'/v2/'.$path.'/manifests/'.$digest;
}
}

50
app/Services/Registries/RegistryDockerAuthScript.php Normal file
View File

@@ -0,0 +1,50 @@
<?php
namespace App\Services\Registries;
use App\Models\Registry;
class RegistryDockerAuthScript
{
/**
* @return array{script: string, secrets: array<string, string>}
*/
public function forBuild(Registry $registry, string $user = 'keystone'): array
{
return $this->forCredential($registry, 'build', $user);
}
/**
* @return array{script: string, secrets: array<string, string>}
*/
public function forRuntime(Registry $registry, string $user = 'keystone'): array
{
return $this->forCredential($registry, 'runtime', $user);
}
/**
* @return array{script: string, secrets: array<string, string>}
*/
private function forCredential(Registry $registry, string $scope, string $user): array
{
$credentials = $registry->credentials ?? [];
$username = (string) ($credentials[$scope.'_username'] ?? $credentials['username'] ?? '');
$password = (string) ($credentials[$scope.'_password'] ?? $credentials['password'] ?? '');
$home = $user === 'root' ? '/root' : '/home/'.$user;
$registryHost = rtrim((string) preg_replace('#^https?://#', '', (string) $registry->url), '/');
return [
'script' => implode("\n", [
'set -euo pipefail',
'install -d -m 700 -o '.escapeshellarg($user).' -g '.escapeshellarg($user).' '.escapeshellarg($home.'/.docker'),
'export DOCKER_CONFIG='.escapeshellarg($home.'/.docker'),
'printf %s '.escapeshellarg('[!registry_password_base64!]').' | base64 -d | docker login '.escapeshellarg($registryHost).' --username '.escapeshellarg($username).' --password-stdin >/dev/null',
'chown '.escapeshellarg($user.':'.$user).' '.escapeshellarg($home.'/.docker/config.json'),
'chmod 600 '.escapeshellarg($home.'/.docker/config.json'),
]),
'secrets' => [
'registry_password_base64' => base64_encode($password),
],
];
}
}

47
app/Services/Registries/RegistryResolver.php Normal file
View File

@@ -0,0 +1,47 @@
<?php
namespace App\Services\Registries;
use App\Enums\RegistryType;
use App\Models\Organisation;
use App\Models\Registry;
class RegistryResolver
{
public function __construct(
private readonly ManagedRegistryHealth $managedRegistryHealth,
) {}
public function buildRegistryFor(Organisation $organisation): ?Registry
{
$externalRegistry = $organisation->registries()
->where('type', '!=', RegistryType::MANAGED->value)
->first();
if ($externalRegistry instanceof Registry) {
return $externalRegistry;
}
$managedRegistry = $organisation->registries()
->where('type', RegistryType::MANAGED->value)
->first();
if ($managedRegistry instanceof Registry) {
return $this->managedRegistryHealth->readinessBlocker($managedRegistry) === null ? $managedRegistry : null;
}
return null;
}
public function managedRegistryBlockerFor(Organisation $organisation): ?string
{
$managedRegistry = $organisation->registries()
->where('type', RegistryType::MANAGED->value)
->with('controlServer')
->first();
return $managedRegistry instanceof Registry
? $this->managedRegistryHealth->readinessBlocker($managedRegistry)
: null;
}
}