Files
ecs-relay-service/deployment-source/scripts/fetch_official_images.py
T

168 lines
8.3 KiB
Python

"""Fetch official linux/arm64 images via Docker's ECR Public official-image mirror.
No daemon or third-party image rebuild is required. Verify registry manifest/blob
digests and uncompressed rootfs diff_ids, then write Docker-load-compatible tar.
"""
import argparse
import concurrent.futures
import gzip
import hashlib
import io
import json
import os
import tarfile
import time
from pathlib import Path
import httpx
REGISTRY = "https://public.ecr.aws"
IMAGES = {"python": "3.12-slim-bookworm", "postgres": "17-alpine",
"redis": "8.2-alpine", "caddy": "2-alpine"}
ACCEPT = ",".join(["application/vnd.oci.image.index.v1+json",
"application/vnd.docker.distribution.manifest.list.v2+json",
"application/vnd.oci.image.manifest.v1+json",
"application/vnd.docker.distribution.manifest.v2+json"])
def digest_file(path):
value = hashlib.sha256()
with path.open("rb") as source:
for chunk in iter(lambda: source.read(4 * 1024 * 1024), b""):
value.update(chunk)
return value.hexdigest()
def checked_response(client, url, headers, expected=None):
for attempt in range(6):
response = client.get(url, headers=headers)
if response.status_code not in {429, 500, 502, 503, 504}:
break
time.sleep(min(2 ** attempt, 12))
response.raise_for_status()
data = response.content
actual = "sha256:" + hashlib.sha256(data).hexdigest()
if expected and actual != expected:
raise RuntimeError("manifest digest mismatch")
return json.loads(data), actual
def blob(name, headers, descriptor, cache):
digest = descriptor["digest"].split(":", 1)[1]
path = cache / (digest + ".blob")
if path.exists() and path.stat().st_size == descriptor["size"] and digest_file(path) == digest:
return path
for attempt in range(5):
try:
temp = path.with_suffix(".part")
# Do not forward the registry bearer token across a blob-storage redirect.
with httpx.Client(timeout=httpx.Timeout(120, connect=20), follow_redirects=False) as client:
url = f"{REGISTRY}/v2/docker/library/{name}/blobs/{descriptor['digest']}"
request_headers = {"Authorization": headers["Authorization"]}
for _ in range(6):
with client.stream("GET", url, headers=request_headers) as response:
if response.is_redirect:
url = str(response.url.join(response.headers["location"]))
request_headers = {} # AWS signed blob URL needs no bearer token.
continue
response.raise_for_status()
value, count = hashlib.sha256(), 0
with temp.open("wb") as output:
for data in response.iter_bytes(1024 * 1024):
output.write(data)
value.update(data)
count += len(data)
if count != descriptor["size"] or value.hexdigest() != digest:
raise RuntimeError("blob integrity mismatch")
temp.replace(path)
print(json.dumps({"downloaded_mb": round(count / 1048576, 1), "blob": digest[:16]}), flush=True)
return path
raise RuntimeError("too many blob redirects")
except (httpx.HTTPError, OSError, RuntimeError) as exc:
if attempt == 4:
raise RuntimeError(f"blob download failed ({type(exc).__name__}); credentials omitted") from None
time.sleep(min(2 ** attempt, 8))
def add_bytes(archive, name, data):
entry = tarfile.TarInfo(name)
entry.size, entry.mode, entry.mtime = len(data), 0o644, 0
archive.addfile(entry, io.BytesIO(data))
def export_image(record, cache, out):
config = record["config"]
cfg_name = config["digest"].split(":")[1] + ".json"
cfg_data = (cache / (config["digest"].split(":")[1] + ".blob")).read_bytes()
cfg = json.loads(cfg_data)
if cfg.get("architecture") != "arm64" or cfg.get("os") != "linux":
raise RuntimeError("image is not linux/arm64")
layers, diffids = [], cfg["rootfs"]["diff_ids"]
if len(diffids) != len(record["layers"]):
raise RuntimeError("rootfs/layer count mismatch")
for desc, expected in zip(record["layers"], diffids):
blobpath = cache / (desc["digest"].split(":")[1] + ".blob")
plain = cache / (expected.split(":")[1] + ".layer.tar")
if not plain.exists() or digest_file(plain) != expected.split(":")[1]:
temp = plain.with_suffix(".part")
with blobpath.open("rb") as source:
magic = source.read(2)
opener = gzip.open if magic == b"\x1f\x8b" else open
value = hashlib.sha256()
with opener(blobpath, "rb") as source, temp.open("wb") as destination:
for chunk in iter(lambda: source.read(4 * 1024 * 1024), b""):
value.update(chunk)
destination.write(chunk)
if "sha256:" + value.hexdigest() != expected:
raise RuntimeError("uncompressed rootfs digest mismatch")
temp.replace(plain)
layers.append((expected.split(":")[1] + "/layer.tar", plain))
output = out / (record["name"] + "-" + record["tag"].replace("/", "-") + "-linux-arm64.tar")
with tarfile.open(output.with_suffix(".part"), "w") as archive:
add_bytes(archive, cfg_name, cfg_data)
for name, path in layers:
archive.add(path, arcname=name, recursive=False)
add_bytes(archive, "manifest.json", json.dumps([{"Config": cfg_name,
"RepoTags": [record["name"] + ":" + record["tag"]],
"Layers": [name for name, _ in layers]}]).encode())
output.with_suffix(".part").replace(output)
record["archive"] = output.name
record["archive_sha256"] = digest_file(output)
record["config_json"] = cfg
print(json.dumps({"exported": output.name, "mb": round(output.stat().st_size / 1048576, 1)}), flush=True)
def main():
p = argparse.ArgumentParser()
p.add_argument("--output", type=Path, required=True)
p.add_argument("--cache", type=Path, required=True)
args = p.parse_args()
args.output.mkdir(parents=True, exist_ok=True)
args.cache.mkdir(parents=True, exist_ok=True)
records = []
with httpx.Client(timeout=30, follow_redirects=False) as client:
for name, tag in IMAGES.items():
token_response = client.get(REGISTRY + "/token/", params={"service": "public.ecr.aws", "scope": f"repository:docker/library/{name}:pull"})
token_response.raise_for_status()
headers = {"Authorization": "Bearer " + token_response.json()["token"], "Accept": ACCEPT}
index, index_digest = checked_response(client, f"{REGISTRY}/v2/docker/library/{name}/manifests/{tag}", headers)
descriptor = next(x for x in index["manifests"] if x.get("platform", {}).get("architecture") == "arm64"
and x.get("platform", {}).get("os") == "linux")
manifest, manifest_digest = checked_response(client, f"{REGISTRY}/v2/docker/library/{name}/manifests/{descriptor['digest']}", headers, descriptor["digest"])
record = {"name": name, "tag": tag, "upstream": f"docker.io/library/{name}:{tag}",
"download_source": f"public.ecr.aws/docker/library/{name}:{tag}",
"platform": "linux/arm64", "index_digest": index_digest,
"manifest_digest": manifest_digest, "config": manifest["config"], "layers": manifest["layers"]}
print(json.dumps({"image": name + ":" + tag, "compressed_mb": round(sum(x['size'] for x in manifest['layers']) / 1048576, 1)}), flush=True)
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
futures = [pool.submit(blob, name, headers, desc, args.cache) for desc in [manifest["config"], *manifest["layers"]]]
for future in futures:
future.result()
export_image(record, args.cache, args.output)
records.append(record)
(args.output / "IMAGE_MANIFEST.json").write_text(json.dumps(records, indent=2) + "\n", encoding="utf-8")
if __name__ == "__main__":
main()