168 lines
8.3 KiB
Python
168 lines
8.3 KiB
Python
"""Fetch official linux/arm64 images via Docker's ECR Public official-image mirror.
|
|
|
|
No daemon or third-party image rebuild is required. Verify registry manifest/blob
|
|
digests and uncompressed rootfs diff_ids, then write Docker-load-compatible tar.
|
|
"""
|
|
import argparse
|
|
import concurrent.futures
|
|
import gzip
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import tarfile
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
|
|
REGISTRY = "https://public.ecr.aws"
|
|
IMAGES = {"python": "3.12-slim-bookworm", "postgres": "17-alpine",
|
|
"redis": "8.2-alpine", "caddy": "2-alpine"}
|
|
ACCEPT = ",".join(["application/vnd.oci.image.index.v1+json",
|
|
"application/vnd.docker.distribution.manifest.list.v2+json",
|
|
"application/vnd.oci.image.manifest.v1+json",
|
|
"application/vnd.docker.distribution.manifest.v2+json"])
|
|
|
|
|
|
def digest_file(path):
|
|
value = hashlib.sha256()
|
|
with path.open("rb") as source:
|
|
for chunk in iter(lambda: source.read(4 * 1024 * 1024), b""):
|
|
value.update(chunk)
|
|
return value.hexdigest()
|
|
|
|
|
|
def checked_response(client, url, headers, expected=None):
|
|
for attempt in range(6):
|
|
response = client.get(url, headers=headers)
|
|
if response.status_code not in {429, 500, 502, 503, 504}:
|
|
break
|
|
time.sleep(min(2 ** attempt, 12))
|
|
response.raise_for_status()
|
|
data = response.content
|
|
actual = "sha256:" + hashlib.sha256(data).hexdigest()
|
|
if expected and actual != expected:
|
|
raise RuntimeError("manifest digest mismatch")
|
|
return json.loads(data), actual
|
|
|
|
|
|
def blob(name, headers, descriptor, cache):
|
|
digest = descriptor["digest"].split(":", 1)[1]
|
|
path = cache / (digest + ".blob")
|
|
if path.exists() and path.stat().st_size == descriptor["size"] and digest_file(path) == digest:
|
|
return path
|
|
for attempt in range(5):
|
|
try:
|
|
temp = path.with_suffix(".part")
|
|
# Do not forward the registry bearer token across a blob-storage redirect.
|
|
with httpx.Client(timeout=httpx.Timeout(120, connect=20), follow_redirects=False) as client:
|
|
url = f"{REGISTRY}/v2/docker/library/{name}/blobs/{descriptor['digest']}"
|
|
request_headers = {"Authorization": headers["Authorization"]}
|
|
for _ in range(6):
|
|
with client.stream("GET", url, headers=request_headers) as response:
|
|
if response.is_redirect:
|
|
url = str(response.url.join(response.headers["location"]))
|
|
request_headers = {} # AWS signed blob URL needs no bearer token.
|
|
continue
|
|
response.raise_for_status()
|
|
value, count = hashlib.sha256(), 0
|
|
with temp.open("wb") as output:
|
|
for data in response.iter_bytes(1024 * 1024):
|
|
output.write(data)
|
|
value.update(data)
|
|
count += len(data)
|
|
if count != descriptor["size"] or value.hexdigest() != digest:
|
|
raise RuntimeError("blob integrity mismatch")
|
|
temp.replace(path)
|
|
print(json.dumps({"downloaded_mb": round(count / 1048576, 1), "blob": digest[:16]}), flush=True)
|
|
return path
|
|
raise RuntimeError("too many blob redirects")
|
|
except (httpx.HTTPError, OSError, RuntimeError) as exc:
|
|
if attempt == 4:
|
|
raise RuntimeError(f"blob download failed ({type(exc).__name__}); credentials omitted") from None
|
|
time.sleep(min(2 ** attempt, 8))
|
|
|
|
|
|
def add_bytes(archive, name, data):
|
|
entry = tarfile.TarInfo(name)
|
|
entry.size, entry.mode, entry.mtime = len(data), 0o644, 0
|
|
archive.addfile(entry, io.BytesIO(data))
|
|
|
|
|
|
def export_image(record, cache, out):
|
|
config = record["config"]
|
|
cfg_name = config["digest"].split(":")[1] + ".json"
|
|
cfg_data = (cache / (config["digest"].split(":")[1] + ".blob")).read_bytes()
|
|
cfg = json.loads(cfg_data)
|
|
if cfg.get("architecture") != "arm64" or cfg.get("os") != "linux":
|
|
raise RuntimeError("image is not linux/arm64")
|
|
layers, diffids = [], cfg["rootfs"]["diff_ids"]
|
|
if len(diffids) != len(record["layers"]):
|
|
raise RuntimeError("rootfs/layer count mismatch")
|
|
for desc, expected in zip(record["layers"], diffids):
|
|
blobpath = cache / (desc["digest"].split(":")[1] + ".blob")
|
|
plain = cache / (expected.split(":")[1] + ".layer.tar")
|
|
if not plain.exists() or digest_file(plain) != expected.split(":")[1]:
|
|
temp = plain.with_suffix(".part")
|
|
with blobpath.open("rb") as source:
|
|
magic = source.read(2)
|
|
opener = gzip.open if magic == b"\x1f\x8b" else open
|
|
value = hashlib.sha256()
|
|
with opener(blobpath, "rb") as source, temp.open("wb") as destination:
|
|
for chunk in iter(lambda: source.read(4 * 1024 * 1024), b""):
|
|
value.update(chunk)
|
|
destination.write(chunk)
|
|
if "sha256:" + value.hexdigest() != expected:
|
|
raise RuntimeError("uncompressed rootfs digest mismatch")
|
|
temp.replace(plain)
|
|
layers.append((expected.split(":")[1] + "/layer.tar", plain))
|
|
output = out / (record["name"] + "-" + record["tag"].replace("/", "-") + "-linux-arm64.tar")
|
|
with tarfile.open(output.with_suffix(".part"), "w") as archive:
|
|
add_bytes(archive, cfg_name, cfg_data)
|
|
for name, path in layers:
|
|
archive.add(path, arcname=name, recursive=False)
|
|
add_bytes(archive, "manifest.json", json.dumps([{"Config": cfg_name,
|
|
"RepoTags": [record["name"] + ":" + record["tag"]],
|
|
"Layers": [name for name, _ in layers]}]).encode())
|
|
output.with_suffix(".part").replace(output)
|
|
record["archive"] = output.name
|
|
record["archive_sha256"] = digest_file(output)
|
|
record["config_json"] = cfg
|
|
print(json.dumps({"exported": output.name, "mb": round(output.stat().st_size / 1048576, 1)}), flush=True)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--output", type=Path, required=True)
|
|
p.add_argument("--cache", type=Path, required=True)
|
|
args = p.parse_args()
|
|
args.output.mkdir(parents=True, exist_ok=True)
|
|
args.cache.mkdir(parents=True, exist_ok=True)
|
|
records = []
|
|
with httpx.Client(timeout=30, follow_redirects=False) as client:
|
|
for name, tag in IMAGES.items():
|
|
token_response = client.get(REGISTRY + "/token/", params={"service": "public.ecr.aws", "scope": f"repository:docker/library/{name}:pull"})
|
|
token_response.raise_for_status()
|
|
headers = {"Authorization": "Bearer " + token_response.json()["token"], "Accept": ACCEPT}
|
|
index, index_digest = checked_response(client, f"{REGISTRY}/v2/docker/library/{name}/manifests/{tag}", headers)
|
|
descriptor = next(x for x in index["manifests"] if x.get("platform", {}).get("architecture") == "arm64"
|
|
and x.get("platform", {}).get("os") == "linux")
|
|
manifest, manifest_digest = checked_response(client, f"{REGISTRY}/v2/docker/library/{name}/manifests/{descriptor['digest']}", headers, descriptor["digest"])
|
|
record = {"name": name, "tag": tag, "upstream": f"docker.io/library/{name}:{tag}",
|
|
"download_source": f"public.ecr.aws/docker/library/{name}:{tag}",
|
|
"platform": "linux/arm64", "index_digest": index_digest,
|
|
"manifest_digest": manifest_digest, "config": manifest["config"], "layers": manifest["layers"]}
|
|
print(json.dumps({"image": name + ":" + tag, "compressed_mb": round(sum(x['size'] for x in manifest['layers']) / 1048576, 1)}), flush=True)
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
|
futures = [pool.submit(blob, name, headers, desc, args.cache) for desc in [manifest["config"], *manifest["layers"]]]
|
|
for future in futures:
|
|
future.result()
|
|
export_image(record, args.cache, args.output)
|
|
records.append(record)
|
|
(args.output / "IMAGE_MANIFEST.json").write_text(json.dumps(records, indent=2) + "\n", encoding="utf-8")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|