"""Fetch official linux/arm64 images via Docker's ECR Public official-image mirror. No daemon or third-party image rebuild is required. Verify registry manifest/blob digests and uncompressed rootfs diff_ids, then write Docker-load-compatible tar. """ import argparse import concurrent.futures import gzip import hashlib import io import json import os import tarfile import time from pathlib import Path import httpx REGISTRY = "https://public.ecr.aws" IMAGES = {"python": "3.12-slim-bookworm", "postgres": "17-alpine", "redis": "8.2-alpine", "caddy": "2-alpine"} ACCEPT = ",".join(["application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json", "application/vnd.oci.image.manifest.v1+json", "application/vnd.docker.distribution.manifest.v2+json"]) def digest_file(path): value = hashlib.sha256() with path.open("rb") as source: for chunk in iter(lambda: source.read(4 * 1024 * 1024), b""): value.update(chunk) return value.hexdigest() def checked_response(client, url, headers, expected=None): for attempt in range(6): response = client.get(url, headers=headers) if response.status_code not in {429, 500, 502, 503, 504}: break time.sleep(min(2 ** attempt, 12)) response.raise_for_status() data = response.content actual = "sha256:" + hashlib.sha256(data).hexdigest() if expected and actual != expected: raise RuntimeError("manifest digest mismatch") return json.loads(data), actual def blob(name, headers, descriptor, cache): digest = descriptor["digest"].split(":", 1)[1] path = cache / (digest + ".blob") if path.exists() and path.stat().st_size == descriptor["size"] and digest_file(path) == digest: return path for attempt in range(5): try: temp = path.with_suffix(".part") # Do not forward the registry bearer token across a blob-storage redirect. with httpx.Client(timeout=httpx.Timeout(120, connect=20), follow_redirects=False) as client: url = f"{REGISTRY}/v2/docker/library/{name}/blobs/{descriptor['digest']}" request_headers = {"Authorization": headers["Authorization"]} for _ in range(6): with client.stream("GET", url, headers=request_headers) as response: if response.is_redirect: url = str(response.url.join(response.headers["location"])) request_headers = {} # AWS signed blob URL needs no bearer token. continue response.raise_for_status() value, count = hashlib.sha256(), 0 with temp.open("wb") as output: for data in response.iter_bytes(1024 * 1024): output.write(data) value.update(data) count += len(data) if count != descriptor["size"] or value.hexdigest() != digest: raise RuntimeError("blob integrity mismatch") temp.replace(path) print(json.dumps({"downloaded_mb": round(count / 1048576, 1), "blob": digest[:16]}), flush=True) return path raise RuntimeError("too many blob redirects") except (httpx.HTTPError, OSError, RuntimeError) as exc: if attempt == 4: raise RuntimeError(f"blob download failed ({type(exc).__name__}); credentials omitted") from None time.sleep(min(2 ** attempt, 8)) def add_bytes(archive, name, data): entry = tarfile.TarInfo(name) entry.size, entry.mode, entry.mtime = len(data), 0o644, 0 archive.addfile(entry, io.BytesIO(data)) def export_image(record, cache, out): config = record["config"] cfg_name = config["digest"].split(":")[1] + ".json" cfg_data = (cache / (config["digest"].split(":")[1] + ".blob")).read_bytes() cfg = json.loads(cfg_data) if cfg.get("architecture") != "arm64" or cfg.get("os") != "linux": raise RuntimeError("image is not linux/arm64") layers, diffids = [], cfg["rootfs"]["diff_ids"] if len(diffids) != len(record["layers"]): raise RuntimeError("rootfs/layer count mismatch") for desc, expected in zip(record["layers"], diffids): blobpath = cache / (desc["digest"].split(":")[1] + ".blob") plain = cache / (expected.split(":")[1] + ".layer.tar") if not plain.exists() or digest_file(plain) != expected.split(":")[1]: temp = plain.with_suffix(".part") with blobpath.open("rb") as source: magic = source.read(2) opener = gzip.open if magic == b"\x1f\x8b" else open value = hashlib.sha256() with opener(blobpath, "rb") as source, temp.open("wb") as destination: for chunk in iter(lambda: source.read(4 * 1024 * 1024), b""): value.update(chunk) destination.write(chunk) if "sha256:" + value.hexdigest() != expected: raise RuntimeError("uncompressed rootfs digest mismatch") temp.replace(plain) layers.append((expected.split(":")[1] + "/layer.tar", plain)) output = out / (record["name"] + "-" + record["tag"].replace("/", "-") + "-linux-arm64.tar") with tarfile.open(output.with_suffix(".part"), "w") as archive: add_bytes(archive, cfg_name, cfg_data) for name, path in layers: archive.add(path, arcname=name, recursive=False) add_bytes(archive, "manifest.json", json.dumps([{"Config": cfg_name, "RepoTags": [record["name"] + ":" + record["tag"]], "Layers": [name for name, _ in layers]}]).encode()) output.with_suffix(".part").replace(output) record["archive"] = output.name record["archive_sha256"] = digest_file(output) record["config_json"] = cfg print(json.dumps({"exported": output.name, "mb": round(output.stat().st_size / 1048576, 1)}), flush=True) def main(): p = argparse.ArgumentParser() p.add_argument("--output", type=Path, required=True) p.add_argument("--cache", type=Path, required=True) args = p.parse_args() args.output.mkdir(parents=True, exist_ok=True) args.cache.mkdir(parents=True, exist_ok=True) records = [] with httpx.Client(timeout=30, follow_redirects=False) as client: for name, tag in IMAGES.items(): token_response = client.get(REGISTRY + "/token/", params={"service": "public.ecr.aws", "scope": f"repository:docker/library/{name}:pull"}) token_response.raise_for_status() headers = {"Authorization": "Bearer " + token_response.json()["token"], "Accept": ACCEPT} index, index_digest = checked_response(client, f"{REGISTRY}/v2/docker/library/{name}/manifests/{tag}", headers) descriptor = next(x for x in index["manifests"] if x.get("platform", {}).get("architecture") == "arm64" and x.get("platform", {}).get("os") == "linux") manifest, manifest_digest = checked_response(client, f"{REGISTRY}/v2/docker/library/{name}/manifests/{descriptor['digest']}", headers, descriptor["digest"]) record = {"name": name, "tag": tag, "upstream": f"docker.io/library/{name}:{tag}", "download_source": f"public.ecr.aws/docker/library/{name}:{tag}", "platform": "linux/arm64", "index_digest": index_digest, "manifest_digest": manifest_digest, "config": manifest["config"], "layers": manifest["layers"]} print(json.dumps({"image": name + ":" + tag, "compressed_mb": round(sum(x['size'] for x in manifest['layers']) / 1048576, 1)}), flush=True) with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: futures = [pool.submit(blob, name, headers, desc, args.cache) for desc in [manifest["config"], *manifest["layers"]]] for future in futures: future.result() export_image(record, args.cache, args.output) records.append(record) (args.output / "IMAGE_MANIFEST.json").write_text(json.dumps(records, indent=2) + "\n", encoding="utf-8") if __name__ == "__main__": main()