Add managed registry provisioning, pruning, and readiness tracking

app/Services/Registries/ImageReference.php

@@ -0,0 +1,54 @@
+<?php
+
+namespace App\Services\Registries;
+
+use App\Enums\RegistryType;
+use App\Models\Environment;
+use App\Models\Registry;
+
+class ImageReference
+{
+    public function tagFor(Environment $environment, string $commitSha, ?Registry $registry = null): string
+    {
+        $tag = substr($commitSha, 0, 12);
+
+        if ($this->registryType($registry) === RegistryType::MANAGED) {
+            $namespace = trim((string) config('keystone.managed_registry.namespace', 'keystone'), '/');
+            $applicationId = $environment->application->uuid ?: 'app-'.$environment->application_id;
+            $environmentId = $environment->uuid ?: 'env-'.$environment->id;
+            $path = $namespace.'/'.$applicationId.'/'.$environmentId;
+
+            return $path.':'.$tag;
+        }
+
+        return str($environment->application->name)
+            ->slug()
+            ->append(':'.$tag)
+            ->value();
+    }
+
+    public function registryReference(Registry $registry, string $imageTag): string
+    {
+        return rtrim($this->registryHost((string) $registry->url), '/').'/'.ltrim($imageTag, '/');
+    }
+
+    private function registryHost(string $url): string
+    {
+        $host = preg_replace('#^https?://#', '', trim($url));
+
+        return $host === null ? trim($url) : $host;
+    }
+
+    private function registryType(?Registry $registry): ?RegistryType
+    {
+        if (! $registry instanceof Registry) {
+            return null;
+        }
+
+        if ($registry->type instanceof RegistryType) {
+            return $registry->type;
+        }
+
+        return RegistryType::tryFrom((string) $registry->type);
+    }
+}

app/Services/Registries/ManagedRegistryHealth.php

@@ -0,0 +1,128 @@
+<?php
+
+namespace App\Services\Registries;
+
+use App\Enums\RegistryType;
+use App\Models\Registry;
+use App\Models\Server;
+use Illuminate\Support\Facades\Http;
+
+class ManagedRegistryHealth
+{
+    public function check(Registry $registry): bool
+    {
+        if ($registry->type !== RegistryType::MANAGED) {
+            return true;
+        }
+
+        $message = $this->configurationBlocker($registry);
+
+        if ($message !== null) {
+            $registry->markUnhealthy($message);
+
+            return false;
+        }
+
+        try {
+            $response = Http::timeout(5)->get('https://'.$registry->url.'/v2/');
+        } catch (\Throwable $exception) {
+            $registry->markUnhealthy('Registry URL is not reachable over HTTPS: '.$exception->getMessage());
+
+            return false;
+        }
+
+        if (! in_array($response->status(), [200, 401], true)) {
+            $registry->markUnhealthy('Registry HTTPS check returned HTTP '.$response->status().'.');
+
+            return false;
+        }
+
+        $checks = $registry->readiness_checks ?? [];
+        $checks['control_https'] = 'passed';
+
+        $registry->forceFill([
+            'readiness_checks' => $checks,
+        ])->save();
+
+        if ($this->readinessChecksPassed($registry->refresh())) {
+            $registry->markHealthy('Registry HTTPS endpoint and smoke checks passed.');
+
+            return true;
+        }
+
+        $registry->markUnhealthy('Registry HTTPS endpoint is reachable, but smoke checks have not all passed.');
+
+        return false;
+    }
+
+    public function readinessBlocker(Registry $registry): ?string
+    {
+        if ($registry->type !== RegistryType::MANAGED) {
+            return null;
+        }
+
+        $message = $this->configurationBlocker($registry);
+
+        if ($message !== null) {
+            return $message;
+        }
+
+        if ($registry->ready_at === null || $registry->health_status !== 'healthy') {
+            return 'Managed registry has not passed readiness checks.';
+        }
+
+        if (! $this->readinessChecksPassed($registry)) {
+            return 'Managed registry smoke checks have not all passed.';
+        }
+
+        return null;
+    }
+
+    private function configurationBlocker(Registry $registry): ?string
+    {
+        if (! $registry->url) {
+            return 'Managed registry URL is not configured.';
+        }
+
+        if (! str_contains((string) $registry->url, '.')) {
+            return 'Managed registry must use a resolvable HTTPS hostname.';
+        }
+
+        $credentials = $registry->credentials ?? [];
+
+        foreach (['build_username', 'build_password', 'runtime_username', 'runtime_password'] as $key) {
+            if (blank($credentials[$key] ?? null)) {
+                return 'Managed registry credentials are incomplete.';
+            }
+        }
+
+        $controlServer = $registry->controlServer;
+
+        if (! $controlServer instanceof Server) {
+            return 'A control/build server is required for managed registry builds.';
+        }
+
+        if (! $controlServer->build_enabled) {
+            return 'The managed registry control server is not build-enabled.';
+        }
+
+        return null;
+    }
+
+    private function readinessChecksPassed(Registry $registry): bool
+    {
+        $checks = $registry->readiness_checks ?? [];
+
+        if ($checks === []) {
+            return false;
+        }
+
+        foreach (['control_https', 'build_push'] as $requiredCheck) {
+            if (! array_key_exists($requiredCheck, $checks)) {
+                return false;
+            }
+        }
+
+        return collect($checks)->every(fn (mixed $status): bool => $status === 'passed');
+    }
+}

app/Services/Registries/ManagedRegistryOperationScripts.php

@@ -0,0 +1,189 @@
+<?php
+
+namespace App\Services\Registries;
+
+use App\Models\BuildArtifact;
+use App\Models\Registry;
+use App\Models\Server;
+
+class ManagedRegistryOperationScripts
+{
+    /**
+     * @return array{script: string, secrets: array<string, string>}
+     */
+    public function provision(Registry $registry): array
+    {
+        $credentials = $registry->credentials ?? [];
+        $host = $this->host($registry);
+        $storagePath = $registry->storage_path ?: (string) config('keystone.managed_registry.storage_path');
+
+        return [
+            'script' => implode("\n", [
+                'set -euo pipefail',
+                'storage_path='.escapeshellarg($storagePath),
+                'registry_host='.escapeshellarg($host),
+                'install -d -m 700 -o root -g root /home/keystone/registry/auth /home/keystone/registry/config',
+                'install -d -m 755 -o root -g root "$storage_path"',
+                'tmp_htpasswd=$(mktemp)',
+                'cleanup() { rm -f "$tmp_htpasswd"; unset build_password runtime_password; }',
+                'trap cleanup EXIT',
+                'build_password=$(printf %s '.escapeshellarg('[!build_password_base64!]').' | base64 -d)',
+                'runtime_password=$(printf %s '.escapeshellarg('[!runtime_password_base64!]').' | base64 -d)',
+                'printf %s "$build_password" | docker run -i --rm --entrypoint htpasswd httpd:2.4-alpine -Bni '.escapeshellarg((string) ($credentials['build_username'] ?? 'keystone-build')).' > "$tmp_htpasswd"',
+                'printf %s "$runtime_password" | docker run -i --rm --entrypoint htpasswd httpd:2.4-alpine -Bni '.escapeshellarg((string) ($credentials['runtime_username'] ?? 'keystone-runtime')).' >> "$tmp_htpasswd"',
+                'install -m 600 -o root -g root "$tmp_htpasswd" /home/keystone/registry/auth/htpasswd',
+                'cat > /home/keystone/registry/config/config.yml <<\'KEYSTONE_REGISTRY_CONFIG\'',
+                'version: 0.1',
+                'log:',
+                '  fields:',
+                '    service: registry',
+                'storage:',
+                '  filesystem:',
+                '    rootdirectory: /var/lib/registry',
+                '  delete:',
+                '    enabled: true',
+                'http:',
+                '  addr: :5000',
+                'auth:',
+                '  htpasswd:',
+                '    realm: keystone-managed-registry',
+                '    path: /auth/htpasswd',
+                'KEYSTONE_REGISTRY_CONFIG',
+                'docker rm -f keystone-managed-registry >/dev/null 2>&1 || true',
+                'docker run -d --name keystone-managed-registry --restart unless-stopped -p 127.0.0.1:5000:5000 -e REGISTRY_STORAGE_DELETE_ENABLED=true -v "$storage_path:/var/lib/registry" -v /home/keystone/registry/auth:/auth:ro -v /home/keystone/registry/config/config.yml:/etc/docker/registry/config.yml:ro registry:2',
+                'install -d -m 755 /home/keystone/gateway/Caddyfile.d',
+                'cat > /home/keystone/gateway/Caddyfile.d/managed-registry.caddy <<KEYSTONE_CADDY_REGISTRY',
+                '$registry_host {',
+                '    reverse_proxy 127.0.0.1:5000',
+                '}',
+                'KEYSTONE_CADDY_REGISTRY',
+                'if test -d /home/keystone/gateway/Caddyfile.d; then cat /home/keystone/gateway/Caddyfile.d/*.caddy > /home/keystone/gateway/Caddyfile; fi',
+                'if docker ps --format \'{{.Names}}\' | grep -qx gateway-1; then docker exec gateway-1 caddy reload --config /etc/caddy/Caddyfile; fi',
+                'if docker ps --format \'{{.Names}}\' | grep -qx caddy; then docker exec caddy caddy reload --config /etc/caddy/Caddyfile; fi',
+                'if docker ps --format \'{{.Names}}\' | grep -Eqx \'(gateway-1|caddy)\'; then curl --fail --silent --show-error --location --head https://"$registry_host"/v2/ || test "$?" = "22"; else echo "Registry proxy reload skipped because no Caddy container is running."; fi',
+            ]),
+            'secrets' => [
+                'build_password_base64' => base64_encode((string) ($credentials['build_password'] ?? '')),
+                'runtime_password_base64' => base64_encode((string) ($credentials['runtime_password'] ?? '')),
+            ],
+        ];
+    }
+
+    /**
+     * @return array{script: string, secrets: array<string, string>}
+     */
+    public function smokeCheck(Registry $registry, Server $server, string $scope, ?string $imageReference = null): array
+    {
+        $credentials = $registry->credentials ?? [];
+        $username = (string) ($credentials[$scope.'_username'] ?? '');
+        $password = (string) ($credentials[$scope.'_password'] ?? '');
+        $host = $this->host($registry);
+        $repository = $imageReference ?: $host.'/keystone/smoke/server-'.$server->id.':latest';
+
+        $commands = [
+            'set -euo pipefail',
+            'registry_host='.escapeshellarg($host),
+            'image_ref='.escapeshellarg($repository),
+            'username='.escapeshellarg($username),
+            'password=$(printf %s '.escapeshellarg('[!registry_password_base64!]').' | base64 -d)',
+            'curl --fail --silent --show-error --user "$username:$password" https://"$registry_host"/v2/ >/dev/null',
+            'printf %s '.escapeshellarg('[!registry_password_base64!]').' | base64 -d | docker login "$registry_host" --username '.escapeshellarg($username).' --password-stdin >/dev/null',
+        ];
+
+        if ($scope === 'build') {
+            $commands = [
+                ...$commands,
+                'docker pull busybox:latest >/dev/null',
+                'docker tag busybox:latest "$image_ref"',
+                'docker push "$image_ref" >/dev/null',
+                'docker buildx imagetools inspect "$image_ref" >/dev/null',
+                'printf "smoke_ref=%s\n" "$image_ref"',
+                'unset password',
+            ];
+        } else {
+            $commands = [
+                ...$commands,
+                'docker pull "$image_ref" >/dev/null',
+                'unset password',
+            ];
+        }
+
+        return [
+            'script' => implode("\n", $commands),
+            'secrets' => [
+                'registry_password_base64' => base64_encode($password),
+            ],
+        ];
+    }
+
+    /**
+     * @param  iterable<int, BuildArtifact>  $artifacts
+     * @return array{script: string, secrets: array<string, string>}
+     */
+    public function maintenance(Registry $registry, iterable $artifacts): array
+    {
+        $credentials = $registry->credentials ?? [];
+        $host = $this->host($registry);
+        $deletions = [];
+
+        foreach ($artifacts as $artifact) {
+            $repository = $this->repositoryPath((string) $artifact->registry_ref);
+
+            if ($repository === '' || blank($artifact->image_digest)) {
+                continue;
+            }
+
+            $deletions[] = 'delete_manifest '.escapeshellarg($repository).' '.escapeshellarg((string) $artifact->image_digest).' || delete_failures=1';
+        }
+
+        return [
+            'script' => implode("\n", [
+                'set -euo pipefail',
+                'registry_host='.escapeshellarg($host),
+                'lock_file=/home/keystone/registry/maintenance.lock',
+                'exec 9>"$lock_file"',
+                'flock -n 9',
+                'username='.escapeshellarg((string) ($credentials['build_username'] ?? 'keystone-build')),
+                'password=$(printf %s '.escapeshellarg('[!registry_password_base64!]').' | base64 -d)',
+                'curl_config=$(mktemp)',
+                'registry_was_stopped=0',
+                'cleanup() { rm -f "$curl_config"; unset password auth_header; if test "$registry_was_stopped" = "1"; then docker start keystone-managed-registry >/dev/null 2>&1 || true; fi; }',
+                'trap cleanup EXIT',
+                'auth_header=$(printf "%s:%s" "$username" "$password" | base64 | tr -d "\n")',
+                'printf "header = \"Authorization: Basic %s\"\n" "$auth_header" > "$curl_config"',
+                'chmod 600 "$curl_config"',
+                'delete_failures=0',
+                'delete_manifest() {',
+                '    repository="$1"',
+                '    digest="$2"',
+                '    status=$(curl --silent --show-error --output /tmp/keystone-registry-delete-response --write-out "%{http_code}" --request DELETE --config "$curl_config" --header "Accept: application/vnd.docker.distribution.manifest.v2+json" "https://$registry_host/v2/$repository/manifests/$digest" || true)',
+                '    case "$status" in 2*|404) printf "deleted_manifest=%s@%s status=%s\n" "$repository" "$digest" "$status" ;; *) cat /tmp/keystone-registry-delete-response >&2; printf "delete_failed=%s@%s status=%s\n" "$repository" "$digest" "$status" >&2; return 1 ;; esac',
+                '}',
+                ...$deletions,
+                'test "$delete_failures" = "0"',
+                'docker stop keystone-managed-registry',
+                'registry_was_stopped=1',
+                'docker run --rm -v /home/keystone/registry/config/config.yml:/etc/docker/registry/config.yml:ro -v '.escapeshellarg(($registry->storage_path ?: (string) config('keystone.managed_registry.storage_path')).':/var/lib/registry').' registry:2 garbage-collect --delete-untagged /etc/docker/registry/config.yml',
+                'docker start keystone-managed-registry',
+                'registry_was_stopped=0',
+                'unset password',
+            ]),
+            'secrets' => [
+                'registry_password_base64' => base64_encode((string) ($credentials['build_password'] ?? '')),
+            ],
+        ];
+    }
+
+    private function host(Registry $registry): string
+    {
+        return rtrim((string) preg_replace('#^https?://#', '', (string) $registry->url), '/');
+    }
+
+    private function repositoryPath(string $reference): string
+    {
+        $withoutHost = str($reference)->after('/')->value();
+        $withoutTag = preg_replace('/:[^:\/]+$/', '', $withoutHost);
+
+        return trim((string) $withoutTag, '/');
+    }
+}

app/Services/Registries/ManagedRegistryProvisioner.php

@@ -0,0 +1,64 @@
+<?php
+
+namespace App\Services\Registries;
+
+use App\Enums\RegistryType;
+use App\Models\Organisation;
+use App\Models\Registry;
+use App\Models\Server;
+use Illuminate\Support\Str;
+
+class ManagedRegistryProvisioner
+{
+    public function provision(Organisation $organisation, string $url, ?Server $controlServer = null, ?string $storagePath = null, ?int $retention = null): Registry
+    {
+        $registry = $organisation->registries()->firstOrNew([
+            'type' => RegistryType::MANAGED->value,
+        ]);
+
+        $registry->fill([
+            'name' => 'Managed Registry',
+            'url' => $this->registryHost($url),
+            'storage_path' => $storagePath ?: (string) config('keystone.managed_registry.storage_path'),
+            'retention_successful_artifacts' => $retention ?: (int) config('keystone.managed_registry.retention.successful_artifacts_per_environment', 3),
+            'control_server_id' => $controlServer?->id,
+            'credentials' => $this->credentials($registry->credentials ?? []),
+        ]);
+
+        if ($registry->health_status === null) {
+            $registry->health_status = 'pending';
+        }
+
+        $registry->save();
+
+        if ($controlServer instanceof Server) {
+            $controlServer->forceFill([
+                'is_control_node' => true,
+                'build_enabled' => true,
+            ])->save();
+        }
+
+        return $registry->refresh();
+    }
+
+    /**
+     * @param  array<string, mixed>  $existing
+     * @return array<string, string>
+     */
+    private function credentials(array $existing): array
+    {
+        return [
+            'build_username' => (string) ($existing['build_username'] ?? 'keystone-build'),
+            'build_password' => (string) ($existing['build_password'] ?? Str::password(40)),
+            'runtime_username' => (string) ($existing['runtime_username'] ?? 'keystone-runtime'),
+            'runtime_password' => (string) ($existing['runtime_password'] ?? Str::password(40)),
+        ];
+    }
+
+    private function registryHost(string $url): string
+    {
+        $host = preg_replace('#^https?://#', '', trim($url));
+
+        return rtrim($host === null ? trim($url) : $host, '/');
+    }
+}

app/Services/Registries/ManagedRegistryRetention.php

@@ -0,0 +1,80 @@
+<?php
+
+namespace App\Services\Registries;
+
+use App\Enums\BuildArtifactStatus;
+use App\Enums\RegistryType;
+use App\Models\BuildArtifact;
+use App\Models\Environment;
+use App\Models\Registry;
+use Illuminate\Support\Collection;
+
+class ManagedRegistryRetention
+{
+    /**
+     * @return Collection<int, BuildArtifact>
+     */
+    public function markPrunable(Registry $registry): Collection
+    {
+        if ($registry->type !== RegistryType::MANAGED) {
+            return collect();
+        }
+
+        $keep = max(1, (int) $registry->retention_successful_artifacts);
+        $updated = collect();
+
+        Environment::query()
+            ->whereHas('buildArtifacts', fn ($query) => $query
+                ->where('status', BuildArtifactStatus::AVAILABLE)
+                ->where('registry_ref', 'like', rtrim((string) $registry->url, '/').'/%'))
+            ->with(['services', 'buildArtifacts' => fn ($query) => $query
+                ->where('status', BuildArtifactStatus::AVAILABLE)
+                ->where('registry_ref', 'like', rtrim((string) $registry->url, '/').'/%')
+                ->latest()])
+            ->each(function (Environment $environment) use ($keep, $updated): void {
+                $activeDigests = $environment->services
+                    ->flatMap(fn ($service): array => [
+                        $service->available_image_digest,
+                        $service->current_image_digest,
+                    ])
+                    ->filter()
+                    ->all();
+
+                $environment->buildArtifacts
+                    ->skip($keep)
+                    ->filter(fn (BuildArtifact $artifact): bool => ! in_array($artifact->image_digest, $activeDigests, true))
+                    ->each(function (BuildArtifact $artifact) use ($updated): void {
+                        $metadata = $artifact->metadata ?? [];
+                        $artifact->update([
+                            'status' => BuildArtifactStatus::PRUNABLE,
+                            'metadata' => [
+                                ...$metadata,
+                                'prunable_at' => now()->toIso8601String(),
+                                'prune_command' => $this->deleteManifestCommand($artifact),
+                            ],
+                        ]);
+                        $updated->push($artifact->refresh());
+                    });
+            });
+
+        return $updated;
+    }
+
+    public function deleteManifestCommand(BuildArtifact $artifact): string
+    {
+        $reference = (string) ($artifact->registry_ref ?? '');
+        $digest = (string) $artifact->image_digest;
+
+        return 'curl --fail --silent --show-error --request DELETE '.escapeshellarg('https://'.$this->manifestPath($reference, $digest));
+    }
+
+    private function manifestPath(string $reference, string $digest): string
+    {
+        $hostAndPath = preg_replace('/:[^:\/]+$/', '', $reference);
+        $path = str($hostAndPath)->after('/')->value();
+
+        return $path === ''
+            ? 'v2/'
+            : str($reference)->before('/')->value().'/v2/'.$path.'/manifests/'.$digest;
+    }
+}

app/Services/Registries/RegistryDockerAuthScript.php

@@ -0,0 +1,50 @@
+<?php
+
+namespace App\Services\Registries;
+
+use App\Models\Registry;
+
+class RegistryDockerAuthScript
+{
+    /**
+     * @return array{script: string, secrets: array<string, string>}
+     */
+    public function forBuild(Registry $registry, string $user = 'keystone'): array
+    {
+        return $this->forCredential($registry, 'build', $user);
+    }
+
+    /**
+     * @return array{script: string, secrets: array<string, string>}
+     */
+    public function forRuntime(Registry $registry, string $user = 'keystone'): array
+    {
+        return $this->forCredential($registry, 'runtime', $user);
+    }
+
+    /**
+     * @return array{script: string, secrets: array<string, string>}
+     */
+    private function forCredential(Registry $registry, string $scope, string $user): array
+    {
+        $credentials = $registry->credentials ?? [];
+        $username = (string) ($credentials[$scope.'_username'] ?? $credentials['username'] ?? '');
+        $password = (string) ($credentials[$scope.'_password'] ?? $credentials['password'] ?? '');
+        $home = $user === 'root' ? '/root' : '/home/'.$user;
+        $registryHost = rtrim((string) preg_replace('#^https?://#', '', (string) $registry->url), '/');
+
+        return [
+            'script' => implode("\n", [
+                'set -euo pipefail',
+                'install -d -m 700 -o '.escapeshellarg($user).' -g '.escapeshellarg($user).' '.escapeshellarg($home.'/.docker'),
+                'export DOCKER_CONFIG='.escapeshellarg($home.'/.docker'),
+                'printf %s '.escapeshellarg('[!registry_password_base64!]').' | base64 -d | docker login '.escapeshellarg($registryHost).' --username '.escapeshellarg($username).' --password-stdin >/dev/null',
+                'chown '.escapeshellarg($user.':'.$user).' '.escapeshellarg($home.'/.docker/config.json'),
+                'chmod 600 '.escapeshellarg($home.'/.docker/config.json'),
+            ]),
+            'secrets' => [
+                'registry_password_base64' => base64_encode($password),
+            ],
+        ];
+    }
+}

app/Services/Registries/RegistryResolver.php

@@ -0,0 +1,47 @@
+<?php
+
+namespace App\Services\Registries;
+
+use App\Enums\RegistryType;
+use App\Models\Organisation;
+use App\Models\Registry;
+
+class RegistryResolver
+{
+    public function __construct(
+        private readonly ManagedRegistryHealth $managedRegistryHealth,
+    ) {}
+
+    public function buildRegistryFor(Organisation $organisation): ?Registry
+    {
+        $externalRegistry = $organisation->registries()
+            ->where('type', '!=', RegistryType::MANAGED->value)
+            ->first();
+
+        if ($externalRegistry instanceof Registry) {
+            return $externalRegistry;
+        }
+
+        $managedRegistry = $organisation->registries()
+            ->where('type', RegistryType::MANAGED->value)
+            ->first();
+
+        if ($managedRegistry instanceof Registry) {
+            return $this->managedRegistryHealth->readinessBlocker($managedRegistry) === null ? $managedRegistry : null;
+        }
+
+        return null;
+    }
+
+    public function managedRegistryBlockerFor(Organisation $organisation): ?string
+    {
+        $managedRegistry = $organisation->registries()
+            ->where('type', RegistryType::MANAGED->value)
+            ->with('controlServer')
+            ->first();
+
+        return $managedRegistry instanceof Registry
+            ? $this->managedRegistryHealth->readinessBlocker($managedRegistry)
+            : null;
+    }
+}