643 lines
31 KiB
Python
643 lines
31 KiB
Python
#!/usr/bin/env python3
|
|
"""Receive and verify one cold backup from the fixed owner SSH session to D:."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime
|
|
import json
|
|
import os
|
|
import re
|
|
import secrets
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
from pathlib import Path
|
|
|
|
from archive import verify_external_archive
|
|
from external_session import (ArchiveFrameWriter, read_challenge, read_response, read_result,
|
|
MAX_ARCHIVE_BYTES, receive_archive, validate_request, write_ack, write_request)
|
|
|
|
DEFAULT_REPOSITORY = Path(r"D:\Dev\akyldash")
|
|
DEFAULT_DEPLOY_SSH_ALIAS = "akyldash_release"
|
|
TRIGGER_CONFIG = Path(r"C:\ProgramData\Akyldash\release-trigger.json")
|
|
AUTO_APPLY_CONFIG = Path(r"C:\ProgramData\Akyldash\release-auto-apply.json")
|
|
PREPARED_STATE_FILE = Path(r"C:\ProgramData\Akyldash\prepared-release.json")
|
|
RELEASE_STATUS_COMMAND = "/usr/local/sbin/akyldash-release-status"
|
|
MAX_STATUS_BYTES = 1024
|
|
STATUS_TIMEOUT_SECONDS = 180
|
|
NOT_READY_REASONS = {
|
|
"recovery_journal", "runtime_config_changed", "release_policy_changed",
|
|
"no_current_main_release", "main_advanced", "already_applied",
|
|
}
|
|
MAX_ERROR_BYTES = 8192
|
|
_WINDOWS = os.name == "nt"
|
|
DEFAULT_SSH = str(Path(r"C:\Windows\System32\OpenSSH\ssh.exe")) if _WINDOWS else "ssh"
|
|
|
|
|
|
def _reparse(path: Path) -> bool:
|
|
if path.is_symlink():
|
|
return True
|
|
try:
|
|
return bool(path.stat().st_file_attributes & 0x400)
|
|
except AttributeError:
|
|
return False
|
|
except FileNotFoundError:
|
|
return False
|
|
|
|
|
|
def _directory_identity(path: Path) -> tuple[int, int]:
|
|
info = path.lstat()
|
|
if _reparse(path) or not stat.S_ISDIR(info.st_mode) or not info.st_ino:
|
|
raise ValueError("backup publication directory is not a stable real directory")
|
|
return info.st_dev, info.st_ino
|
|
|
|
|
|
def _remove_owned_directory(path: Path, identity: tuple[int, int]) -> None:
|
|
try:
|
|
current = _directory_identity(path)
|
|
except FileNotFoundError:
|
|
return
|
|
if current != identity:
|
|
raise ValueError(f"refusing to remove a replaced backup directory: {path}")
|
|
for child in path.iterdir():
|
|
child_info = child.lstat()
|
|
if (child.name not in {"backup.tar.gz", "backup.tar.gz.sha256"}
|
|
or _reparse(child) or not stat.S_ISREG(child_info.st_mode)):
|
|
raise ValueError(f"refusing to remove unexpected backup content: {child}")
|
|
shutil.rmtree(path)
|
|
|
|
|
|
def check_private_acl(path: Path) -> None:
|
|
command = r"""
|
|
$ErrorActionPreference = 'Stop'
|
|
$acl = Get-Acl -LiteralPath $env:AKYLDASH_ACL_PATH
|
|
$driveRoot = [System.IO.Path]::GetPathRoot($env:AKYLDASH_ACL_PATH)
|
|
$volume = [System.IO.DriveInfo]::new($driveRoot)
|
|
if (-not $volume.IsReady -or $volume.DriveFormat -ne 'NTFS') { exit 5 }
|
|
$owner = [System.Security.Principal.WindowsIdentity]::GetCurrent().User.Value
|
|
$allowed = @($owner, 'S-1-5-18', 'S-1-5-32-544')
|
|
$aclOwner = $acl.GetOwner([System.Security.Principal.SecurityIdentifier]).Value
|
|
if ($aclOwner -notin $allowed) { exit 6 }
|
|
$ownerAllowed = $false
|
|
$rules = $acl.GetAccessRules($true, $true, [System.Security.Principal.SecurityIdentifier])
|
|
foreach ($rule in $rules) {
|
|
$sid = $rule.IdentityReference.Value
|
|
if ($rule.AccessControlType -ne [System.Security.AccessControl.AccessControlType]::Allow -or $sid -notin $allowed) { exit 2 }
|
|
if (($rule.FileSystemRights -band [System.Security.AccessControl.FileSystemRights]::FullControl) -ne [System.Security.AccessControl.FileSystemRights]::FullControl) { exit 3 }
|
|
if ($sid -eq $owner) { $ownerAllowed = $true }
|
|
}
|
|
if (-not $ownerAllowed) { exit 4 }
|
|
"""
|
|
environment = {
|
|
"SystemRoot": os.environ.get("SystemRoot", r"C:\Windows"),
|
|
"AKYLDASH_ACL_PATH": str(path),
|
|
}
|
|
result = subprocess.run(
|
|
["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", command],
|
|
env=environment, text=True, capture_output=True,
|
|
)
|
|
if result.returncode:
|
|
raise ValueError(f"backup path ACL is not private (PowerShell status {result.returncode})")
|
|
|
|
|
|
def check_capacity(free_bytes: int) -> None:
|
|
if free_bytes < MAX_ARCHIVE_BYTES + 1024 * 1024:
|
|
raise ValueError("D: must have room for the maximum 100 GiB archive plus staging metadata")
|
|
|
|
|
|
def check_repository_target(repository: Path, backup_id: str, *, allow_existing: bool = False) -> tuple[Path, Path]:
|
|
if not repository.is_absolute() or not repository.is_dir() or _reparse(repository):
|
|
raise ValueError("repository must be an existing real directory on the selected Windows volume")
|
|
if _WINDOWS and repository.drive.upper() != "D:":
|
|
raise ValueError("external backup destination is restricted to D:\\Dev\\akyldash")
|
|
if _WINDOWS and repository.resolve().as_posix().lower() != "d:/dev/akyldash":
|
|
raise ValueError("external backup destination is restricted to D:\\Dev\\akyldash")
|
|
if _WINDOWS:
|
|
check_capacity(shutil.disk_usage(repository).free)
|
|
for parent in repository.parents:
|
|
if _reparse(parent):
|
|
raise ValueError("backup destination has a reparse-point parent")
|
|
git_dir = subprocess.run(
|
|
["git", "-C", str(repository), "rev-parse", "--absolute-git-dir"],
|
|
check=True, text=True, capture_output=True,
|
|
).stdout.strip()
|
|
exclude = Path(git_dir) / "info" / "exclude"
|
|
if _reparse(exclude) or not exclude.is_file():
|
|
raise ValueError("the repository's local .git/info/exclude file is unavailable or unsafe")
|
|
root = repository / ".backups"
|
|
final = root / backup_id
|
|
target_file = final / "backup.tar.gz"
|
|
relative = target_file.relative_to(repository).as_posix()
|
|
result = subprocess.run(
|
|
["git", "-C", str(repository), "check-ignore", "--no-index", "-v", "--", relative],
|
|
text=True, capture_output=True,
|
|
)
|
|
if result.returncode:
|
|
raise ValueError("the exact D: backup target is not ignored by local Git metadata")
|
|
try:
|
|
source, _line, pattern = result.stdout.split("\t", 1)[0].rsplit(":", 2)
|
|
except ValueError as error:
|
|
raise ValueError("git check-ignore returned an unrecognized source record") from error
|
|
source_path = Path(source)
|
|
if not source_path.is_absolute():
|
|
source_path = repository / source_path
|
|
if source_path.resolve() != exclude.resolve() or pattern != "/.backups/":
|
|
raise ValueError("the exact D: backup target must be ignored by .git/info/exclude")
|
|
if root.exists():
|
|
if _reparse(root) or not root.is_dir():
|
|
raise ValueError("D: backup root must be a real directory")
|
|
else:
|
|
root.mkdir()
|
|
check_private_acl(root)
|
|
if final.exists() or _reparse(final):
|
|
if not allow_existing or _reparse(final) or not final.is_dir():
|
|
raise ValueError("backup ID already exists or resolves through a reparse point")
|
|
elif allow_existing:
|
|
raise ValueError("the pinned external backup is missing from D:")
|
|
return root, final
|
|
|
|
|
|
def open_previous_backup(repository: Path, challenge: dict) -> Path:
|
|
_root, directory = check_repository_target(repository, challenge["backup_id"], allow_existing=True)
|
|
check_private_acl(directory)
|
|
archive_path = directory / "backup.tar.gz"
|
|
sidecar = directory / "backup.tar.gz.sha256"
|
|
for path in (archive_path, sidecar):
|
|
if _reparse(path) or not path.is_file():
|
|
raise ValueError("pinned external backup files must be regular files")
|
|
check_private_acl(path)
|
|
if archive_path.stat().st_size != challenge["size"]:
|
|
raise ValueError("pinned external backup size does not match root-owned release state")
|
|
expected_sidecar = f"{challenge['archive_sha256']} backup.tar.gz\n"
|
|
if sidecar.read_text(encoding="ascii") != expected_sidecar:
|
|
raise ValueError("pinned external backup checksum sidecar does not match root-owned release state")
|
|
manifest = verify_external_archive(
|
|
archive_path,
|
|
expected_sha256=challenge["archive_sha256"],
|
|
expected_manifest_sha256=challenge["manifest_sha256"],
|
|
)
|
|
if manifest != challenge["manifest"]:
|
|
raise ValueError("pinned external backup contents differ from root-owned release state")
|
|
return archive_path
|
|
|
|
|
|
def _drain_stderr(stream, output: bytearray) -> None:
|
|
while True:
|
|
chunk = stream.read(1024)
|
|
if not chunk:
|
|
return
|
|
if len(output) < MAX_ERROR_BYTES:
|
|
output.extend(chunk[: MAX_ERROR_BYTES - len(output)])
|
|
|
|
|
|
def _unique_json_object(pairs: list[tuple[str, object]]) -> dict:
|
|
result = {}
|
|
for key, value in pairs:
|
|
if key in result:
|
|
raise ValueError("release status contains duplicate JSON fields")
|
|
result[key] = value
|
|
return result
|
|
|
|
|
|
def validate_status_response(payload: bytes) -> dict | None:
|
|
if not payload or len(payload) > MAX_STATUS_BYTES:
|
|
raise ValueError("release status response is empty or exceeds its size limit")
|
|
try:
|
|
value = json.loads(payload.decode("utf-8"), object_pairs_hook=_unique_json_object)
|
|
except (UnicodeError, json.JSONDecodeError) as error:
|
|
raise ValueError("release status response is invalid JSON") from error
|
|
if (not isinstance(value, dict) or set(value) != {"schema_version", "state", "reason", "release"}
|
|
or type(value["schema_version"]) is not int or value["schema_version"] != 1):
|
|
raise ValueError("release status response has an unsupported schema")
|
|
if value["state"] == "not_ready":
|
|
if (not isinstance(value["reason"], str) or value["reason"] not in NOT_READY_REASONS
|
|
or value["release"] is not None):
|
|
raise ValueError("release status not-ready response is invalid")
|
|
return None
|
|
if value["state"] != "prepared" or value["reason"] is not None:
|
|
raise ValueError("release status state is invalid")
|
|
release = value["release"]
|
|
if not isinstance(release, dict) or set(release) != {"run_id", "commit", "descriptor_sha256"}:
|
|
raise ValueError("release status candidate tuple is invalid")
|
|
if (not isinstance(release["run_id"], str) or not re.fullmatch(r"[0-9]{1,20}", release["run_id"])
|
|
or not isinstance(release["commit"], str) or not re.fullmatch(r"[0-9a-f]{40}", release["commit"])
|
|
or not isinstance(release["descriptor_sha256"], str)
|
|
or not re.fullmatch(r"[0-9a-f]{64}", release["descriptor_sha256"])):
|
|
raise ValueError("release status candidate identity is invalid")
|
|
request = {"version": 1, "action": "release", **release}
|
|
validate_request(request)
|
|
return request
|
|
|
|
|
|
def write_prepared_request(request: dict, path: Path = PREPARED_STATE_FILE) -> None:
|
|
validate_request(request)
|
|
if _reparse(path.parent) or not path.parent.is_dir() or _reparse(path):
|
|
raise ValueError("prepared release state path is missing or unsafe")
|
|
check_private_acl(path.parent)
|
|
if path.exists():
|
|
check_private_acl(path)
|
|
value = {
|
|
"schema_version": 1,
|
|
"prepared_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
"release": {key: request[key] for key in ("run_id", "commit", "descriptor_sha256")},
|
|
}
|
|
payload = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n"
|
|
temporary = path.with_name(f".{path.name}.{secrets.token_hex(12)}.tmp")
|
|
try:
|
|
with temporary.open("xb") as stream:
|
|
stream.write(payload)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
check_private_acl(temporary)
|
|
os.replace(temporary, path)
|
|
check_private_acl(path)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def read_prepared_request(path: Path = PREPARED_STATE_FILE) -> dict:
|
|
if _reparse(path.parent) or _reparse(path) or not path.parent.is_dir() or not path.is_file():
|
|
raise ValueError("prepared release state is missing or unsafe")
|
|
check_private_acl(path.parent)
|
|
check_private_acl(path)
|
|
raw = path.read_bytes()
|
|
if not raw or len(raw) > MAX_STATUS_BYTES:
|
|
raise ValueError("prepared release state is empty or oversized")
|
|
try:
|
|
value = json.loads(raw.decode("utf-8"), object_pairs_hook=_unique_json_object)
|
|
except (UnicodeError, json.JSONDecodeError) as error:
|
|
raise ValueError("prepared release state is invalid JSON") from error
|
|
if (not isinstance(value, dict) or set(value) != {"schema_version", "prepared_at", "release"}
|
|
or type(value["schema_version"]) is not int or value["schema_version"] != 1
|
|
or not isinstance(value["prepared_at"], str) or not isinstance(value["release"], dict)):
|
|
raise ValueError("prepared release state has an unsupported schema")
|
|
release = value["release"]
|
|
if set(release) != {"run_id", "commit", "descriptor_sha256"}:
|
|
raise ValueError("prepared release tuple is invalid")
|
|
request = {"version": 1, "action": "release", **release}
|
|
validate_request(request)
|
|
return request
|
|
|
|
|
|
def load_trigger_config(path: Path = TRIGGER_CONFIG) -> dict:
|
|
if not _WINDOWS:
|
|
raise ValueError("automatic release trigger is available only on Windows")
|
|
if (_reparse(path) or _reparse(path.parent) or not path.is_file()
|
|
or not path.parent.is_dir()):
|
|
raise ValueError("release trigger configuration is missing or unsafe")
|
|
check_private_acl(path.parent)
|
|
check_private_acl(path)
|
|
try:
|
|
with path.open("rb") as stream:
|
|
raw = stream.read(4097)
|
|
except OSError as error:
|
|
raise ValueError("release trigger configuration is unreadable") from error
|
|
if not raw or len(raw) > 4096:
|
|
raise ValueError("release trigger configuration is empty or oversized")
|
|
try:
|
|
value = json.loads(raw.decode("utf-8"), object_pairs_hook=_unique_json_object)
|
|
except (UnicodeError, json.JSONDecodeError) as error:
|
|
raise ValueError("release trigger configuration is invalid JSON") from error
|
|
if (not isinstance(value, dict)
|
|
or set(value) != {"schema_version", "enabled", "repository", "ssh_alias"}
|
|
or type(value["schema_version"]) is not int or value["schema_version"] != 1
|
|
or type(value["enabled"]) is not bool
|
|
or value["repository"] != str(DEFAULT_REPOSITORY)
|
|
or value["ssh_alias"] != DEFAULT_DEPLOY_SSH_ALIAS):
|
|
raise ValueError("release trigger configuration does not match the fixed deployment target")
|
|
return value
|
|
|
|
|
|
def load_auto_apply_config(path: Path = AUTO_APPLY_CONFIG) -> dict:
|
|
if not _WINDOWS:
|
|
raise ValueError("automatic release apply is available only on Windows")
|
|
if (_reparse(path) or _reparse(path.parent) or not path.is_file() or not path.parent.is_dir()):
|
|
raise ValueError("automatic release apply configuration is missing or unsafe")
|
|
check_private_acl(path.parent)
|
|
check_private_acl(path)
|
|
try:
|
|
raw = path.read_bytes()
|
|
except OSError as error:
|
|
raise ValueError("automatic release apply configuration is unreadable") from error
|
|
if not raw or len(raw) > 1024:
|
|
raise ValueError("automatic release apply configuration is empty or oversized")
|
|
try:
|
|
value = json.loads(raw.decode("utf-8"), object_pairs_hook=_unique_json_object)
|
|
except (UnicodeError, json.JSONDecodeError) as error:
|
|
raise ValueError("automatic release apply configuration is invalid JSON") from error
|
|
if (not isinstance(value, dict) or set(value) != {"schema_version", "enabled"}
|
|
or type(value["schema_version"]) is not int or value["schema_version"] != 1
|
|
or type(value["enabled"]) is not bool):
|
|
raise ValueError("automatic release apply configuration is invalid")
|
|
return value
|
|
|
|
|
|
def query_release_status(*, ssh: str = DEFAULT_SSH, ssh_alias: str = DEFAULT_DEPLOY_SSH_ALIAS) -> dict | None:
|
|
if shutil.which(ssh) is None:
|
|
raise ValueError("OpenSSH client is unavailable")
|
|
if ssh_alias != DEFAULT_DEPLOY_SSH_ALIAS:
|
|
raise ValueError("automatic release status uses only the fixed deployment SSH alias")
|
|
command = [
|
|
ssh, "-T", "-o", "BatchMode=yes", "-o", "ConnectTimeout=15",
|
|
"-o", "ClearAllForwardings=yes", "-o", "ForwardAgent=no", "-o", "ForwardX11=no",
|
|
ssh_alias, RELEASE_STATUS_COMMAND,
|
|
]
|
|
process = subprocess.Popen(
|
|
command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
)
|
|
output = bytearray()
|
|
overflow = threading.Event()
|
|
|
|
def read_status() -> None:
|
|
while True:
|
|
chunk = process.stdout.read(min(4096, MAX_STATUS_BYTES + 1 - len(output)))
|
|
if not chunk:
|
|
return
|
|
if len(output) + len(chunk) > MAX_STATUS_BYTES:
|
|
overflow.set()
|
|
process.kill()
|
|
return
|
|
output.extend(chunk)
|
|
|
|
stdout_thread = threading.Thread(target=read_status, daemon=True)
|
|
stderr_thread = threading.Thread(target=_drain_stderr, args=(process.stderr, bytearray()), daemon=True)
|
|
stdout_thread.start()
|
|
stderr_thread.start()
|
|
try:
|
|
return_code = process.wait(timeout=STATUS_TIMEOUT_SECONDS)
|
|
except subprocess.TimeoutExpired as error:
|
|
process.kill()
|
|
process.wait()
|
|
raise ValueError("release status request timed out") from error
|
|
finally:
|
|
stdout_thread.join()
|
|
stderr_thread.join()
|
|
if overflow.is_set() or return_code:
|
|
raise ValueError("release status command failed or exceeded its response limit")
|
|
return validate_status_response(bytes(output))
|
|
|
|
|
|
def receive_backup(reader, writer, *, repository: Path, request: dict, expected_context: dict | None = None) -> dict:
|
|
frame, value = read_response(reader)
|
|
if frame == "result":
|
|
raise ValueError(value["message"] or "remote deploy preflight failed")
|
|
header = value
|
|
context = header["context"]
|
|
if request["action"] == "release":
|
|
expected_context = {key: request[key] for key in ("action", "run_id", "commit", "descriptor_sha256")}
|
|
if context != expected_context:
|
|
raise ValueError("source backup session does not match the requested release tuple")
|
|
elif expected_context is None or context != expected_context:
|
|
raise ValueError("rollback backup session does not match the pinned previous backup")
|
|
|
|
root, final = check_repository_target(repository, header["backup_id"])
|
|
staging = Path(tempfile.mkdtemp(prefix=".staging-", dir=root))
|
|
staging_identity = None
|
|
published_identity = None
|
|
ack_started = False
|
|
try:
|
|
staging_identity = _directory_identity(staging)
|
|
check_private_acl(staging)
|
|
archive_path = staging / "backup.tar.gz"
|
|
with archive_path.open("xb") as archive_file:
|
|
size, archive_sha256 = receive_archive(reader, archive_file)
|
|
archive_file.flush()
|
|
os.fsync(archive_file.fileno())
|
|
os.chmod(archive_path, 0o600)
|
|
check_private_acl(archive_path)
|
|
manifest = verify_external_archive(
|
|
archive_path,
|
|
expected_sha256=archive_sha256,
|
|
expected_manifest_sha256=header["manifest_sha256"],
|
|
)
|
|
if manifest != header["manifest"]:
|
|
raise ValueError("saved backup contents differ from the source-generated session manifest")
|
|
sidecar = staging / "backup.tar.gz.sha256"
|
|
with sidecar.open("x", encoding="ascii", newline="\n") as stream:
|
|
stream.write(f"{archive_sha256} backup.tar.gz\n")
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.chmod(sidecar, 0o600)
|
|
check_private_acl(sidecar)
|
|
if final.exists() or _reparse(final):
|
|
raise ValueError("backup ID already exists or resolves through a reparse point")
|
|
os.rename(staging, final)
|
|
published_identity = staging_identity
|
|
if _directory_identity(final) != published_identity:
|
|
raise ValueError("published backup directory identity changed during rename")
|
|
for path in (final, final / "backup.tar.gz", final / "backup.tar.gz.sha256"):
|
|
check_private_acl(path)
|
|
ack_started = True
|
|
write_ack(
|
|
writer, session_nonce=header["nonce"], backup_id=header["backup_id"], size=size,
|
|
archive_sha256=archive_sha256, manifest_sha256=header["manifest_sha256"], verified=True,
|
|
)
|
|
return {"backup_id": header["backup_id"], "archive_sha256": archive_sha256, "size": size}
|
|
except Exception as error:
|
|
cleanup_path = final if published_identity is not None and not ack_started else staging
|
|
cleanup_identity = published_identity if cleanup_path == final else staging_identity
|
|
if cleanup_identity is not None:
|
|
try:
|
|
_remove_owned_directory(cleanup_path, cleanup_identity)
|
|
except Exception as cleanup_error:
|
|
raise ValueError(
|
|
f"backup failed and its output could not be safely removed; inspect {cleanup_path}"
|
|
) from cleanup_error
|
|
raise
|
|
|
|
|
|
def run_release(request: dict, *, repository: Path = DEFAULT_REPOSITORY, ssh: str = DEFAULT_SSH,
|
|
ssh_alias: str = "akyldash_vps") -> None:
|
|
if shutil.which(ssh) is None:
|
|
raise ValueError("OpenSSH client is unavailable")
|
|
if not ssh_alias or any(character not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.-" for character in ssh_alias):
|
|
raise ValueError("SSH alias must name an existing owner-controlled host entry")
|
|
check_repository_target(repository, "preflight-" + secrets.token_hex(16))
|
|
process = subprocess.Popen(
|
|
[ssh, "-T", "-o", "BatchMode=yes", "-o", "ConnectTimeout=15",
|
|
"-o", "ClearAllForwardings=yes", "-o", "ForwardAgent=no", "-o", "ForwardX11=no",
|
|
ssh_alias, "/usr/local/sbin/akyldash-release-deploy"],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
)
|
|
errors = bytearray()
|
|
thread = threading.Thread(target=_drain_stderr, args=(process.stderr, errors), daemon=True)
|
|
thread.start()
|
|
try:
|
|
write_request(process.stdin, request)
|
|
process.stdin.flush()
|
|
saved = receive_backup(process.stdout, process.stdin, repository=repository, request=request)
|
|
final = read_result(process.stdout)
|
|
if process.stdout.read(1):
|
|
raise ValueError("unexpected trailing bytes after external deployment result")
|
|
process.stdin.close()
|
|
return_code = process.wait()
|
|
thread.join()
|
|
if return_code or final["status"] != "success":
|
|
message = final.get("message", "remote deployment failed")
|
|
raise ValueError(f"remote deployment failed (ssh status {return_code}): {message}")
|
|
print(f"PASS release completed; verified external backup {saved['backup_id']} ({saved['size']} bytes)")
|
|
except Exception:
|
|
if process.poll() is None:
|
|
process.kill()
|
|
process.wait()
|
|
thread.join()
|
|
raise
|
|
|
|
|
|
def run_release_once(*, config_file: Path = TRIGGER_CONFIG, prepared_file: Path = PREPARED_STATE_FILE,
|
|
ssh: str = DEFAULT_SSH) -> None:
|
|
config = load_trigger_config(config_file)
|
|
if not config["enabled"]:
|
|
print("Release trigger is disabled; no SSH connection opened.")
|
|
return
|
|
request = query_release_status(ssh=ssh, ssh_alias=config["ssh_alias"])
|
|
if request is None:
|
|
print("No verified current-main release is prepared; no deployment started.")
|
|
return
|
|
write_prepared_request(request, prepared_file)
|
|
print(f"PASS prepared release {request['commit']} (run {request['run_id']}); owner confirmation is required to deploy.")
|
|
|
|
|
|
def run_auto_apply_once(*, config_file: Path = TRIGGER_CONFIG,
|
|
auto_apply_file: Path = AUTO_APPLY_CONFIG,
|
|
prepared_file: Path = PREPARED_STATE_FILE,
|
|
ssh: str = DEFAULT_SSH) -> None:
|
|
config = load_trigger_config(config_file)
|
|
auto_config = load_auto_apply_config(auto_apply_file)
|
|
if not config["enabled"] or not auto_config["enabled"]:
|
|
print("Automatic apply is disabled; no SSH connection opened.")
|
|
return
|
|
request = query_release_status(ssh=ssh, ssh_alias=config["ssh_alias"])
|
|
if request is None:
|
|
print("No verified current-main release is prepared; no deployment started.")
|
|
return
|
|
if _reparse(prepared_file):
|
|
raise ValueError("prepared release state is unsafe")
|
|
if not prepared_file.exists():
|
|
write_prepared_request(request, prepared_file)
|
|
print(f"Prepared release {request['commit']}; automatic apply requires a later poll.")
|
|
return
|
|
previous = read_prepared_request(prepared_file)
|
|
if previous != request:
|
|
write_prepared_request(request, prepared_file)
|
|
print(f"Prepared release {request['commit']}; automatic apply requires a later poll.")
|
|
return
|
|
run_release(request, repository=Path(config["repository"]), ssh=ssh, ssh_alias=config["ssh_alias"])
|
|
|
|
|
|
def apply_prepared_release(*, config_file: Path = TRIGGER_CONFIG, prepared_file: Path = PREPARED_STATE_FILE,
|
|
ssh: str = DEFAULT_SSH, confirm=input) -> None:
|
|
if not sys.stdin.isatty():
|
|
raise ValueError("applying a prepared release requires an interactive owner confirmation")
|
|
request = read_prepared_request(prepared_file)
|
|
answer = confirm(f"Type APPLY {request['commit']} to start the separately approved production release: ")
|
|
if answer != f"APPLY {request['commit']}":
|
|
print("Prepared release was not applied.")
|
|
return
|
|
config = load_trigger_config(config_file)
|
|
run_release(request, repository=Path(config["repository"]), ssh=ssh, ssh_alias=config["ssh_alias"])
|
|
|
|
|
|
def run_rollback(*, repository: Path = DEFAULT_REPOSITORY, ssh: str = DEFAULT_SSH,
|
|
ssh_alias: str = "akyldash_vps") -> None:
|
|
if shutil.which(ssh) is None:
|
|
raise ValueError("OpenSSH client is unavailable")
|
|
if not ssh_alias or any(character not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.-" for character in ssh_alias):
|
|
raise ValueError("SSH alias must name an existing owner-controlled host entry")
|
|
check_repository_target(repository, "preflight-" + secrets.token_hex(16))
|
|
process = subprocess.Popen(
|
|
[ssh, "-T", ssh_alias, "/usr/local/sbin/akyldash-release-rollback"],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
)
|
|
errors = bytearray()
|
|
thread = threading.Thread(target=_drain_stderr, args=(process.stderr, errors), daemon=True)
|
|
thread.start()
|
|
request = {"version": 1, "action": "rollback", "data_compatible": True}
|
|
try:
|
|
write_request(process.stdin, request)
|
|
process.stdin.flush()
|
|
challenge = read_challenge(process.stdout)
|
|
archive_path = open_previous_backup(repository, challenge)
|
|
framed = ArchiveFrameWriter(process.stdin, maximum=challenge["size"])
|
|
with archive_path.open("rb") as archive_file:
|
|
for chunk in iter(lambda: archive_file.read(1024 * 1024), b""):
|
|
framed.write(chunk)
|
|
size, digest = framed.finish()
|
|
if size != challenge["size"] or digest != challenge["archive_sha256"]:
|
|
raise ValueError("D: previous backup changed during the live rollback session")
|
|
write_ack(process.stdin, session_nonce=challenge["nonce"], backup_id=challenge["backup_id"],
|
|
size=size, archive_sha256=digest, manifest_sha256=challenge["manifest_sha256"], verified=True)
|
|
receive_backup(process.stdout, process.stdin, repository=repository, request=request,
|
|
expected_context={key: challenge[key] for key in
|
|
("backup_id", "archive_sha256", "manifest_sha256")} | {"action": "rollback"})
|
|
final = read_result(process.stdout)
|
|
if process.stdout.read(1):
|
|
raise ValueError("unexpected trailing bytes after external rollback result")
|
|
process.stdin.close()
|
|
return_code = process.wait()
|
|
thread.join()
|
|
if return_code or final["status"] != "success":
|
|
raise ValueError(f"remote rollback failed (ssh status {return_code}): {final.get('message', 'error')}")
|
|
print(f"PASS rollback completed; fresh external backup saved for {challenge['backup_id']}")
|
|
except Exception:
|
|
if process.poll() is None:
|
|
process.kill()
|
|
process.wait()
|
|
thread.join()
|
|
raise
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--once", action="store_true", help="prepare at most one verified release; never deploy")
|
|
parser.add_argument("--apply-prepared", action="store_true", help="deploy the prepared tuple after interactive owner confirmation")
|
|
parser.add_argument("--auto-apply-prepared", action="store_true", help="apply an already prepared tuple only when the separate owner auto-apply gate is enabled")
|
|
parser.add_argument("--rollback", action="store_true")
|
|
parser.add_argument("--data-compatible", action="store_true")
|
|
parser.add_argument("--run-id")
|
|
parser.add_argument("--commit")
|
|
parser.add_argument("--descriptor-sha256")
|
|
parser.add_argument("--ssh-alias", default="akyldash_vps")
|
|
args = parser.parse_args()
|
|
try:
|
|
if args.once:
|
|
if (args.rollback or args.data_compatible or args.ssh_alias != "akyldash_vps" or any(
|
|
(args.run_id, args.commit, args.descriptor_sha256))
|
|
):
|
|
raise ValueError("--once accepts no release tuple, rollback, compatibility flag, or custom SSH alias")
|
|
run_release_once()
|
|
elif args.apply_prepared:
|
|
if (args.rollback or args.data_compatible or args.ssh_alias != "akyldash_vps"
|
|
or any((args.run_id, args.commit, args.descriptor_sha256))):
|
|
raise ValueError("--apply-prepared accepts no release tuple or rollback flags")
|
|
apply_prepared_release()
|
|
elif args.auto_apply_prepared:
|
|
if (args.rollback or args.data_compatible or args.ssh_alias != "akyldash_vps"
|
|
or any((args.run_id, args.commit, args.descriptor_sha256))):
|
|
raise ValueError("--auto-apply-prepared accepts no release tuple or rollback flags")
|
|
run_auto_apply_once()
|
|
elif args.rollback:
|
|
if not args.data_compatible or any((args.run_id, args.commit, args.descriptor_sha256)):
|
|
raise ValueError("rollback requires --data-compatible and accepts no release tuple")
|
|
run_rollback(ssh_alias=args.ssh_alias)
|
|
else:
|
|
if not args.run_id or not args.commit or not args.descriptor_sha256 or args.data_compatible:
|
|
raise ValueError("release requires --run-id, --commit, and --descriptor-sha256")
|
|
request = {
|
|
"version": 1, "action": "release", "run_id": args.run_id,
|
|
"commit": args.commit, "descriptor_sha256": args.descriptor_sha256,
|
|
}
|
|
run_release(request, ssh_alias=args.ssh_alias)
|
|
except (OSError, ValueError, subprocess.SubprocessError, json.JSONDecodeError) as error:
|
|
print(f"ERROR: {error}", file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|