Add self-contained relay deployment project and operational guidance
This commit is contained in:
@@ -0,0 +1,140 @@
|
||||
"""Initialize a NEW deployment only. Run as root inside the application image."""
|
||||
import argparse
|
||||
import ipaddress
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import ssl
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import ed25519
|
||||
from cryptography.x509.oid import NameOID
|
||||
from dotenv import dotenv_values, set_key
|
||||
from relay.crypto import issue_ca, pem_private
|
||||
|
||||
|
||||
def initialize(root, test_cert=False):
|
||||
root = Path(root)
|
||||
if os.name != 'nt' and os.geteuid() != 0:
|
||||
raise ValueError('Run initialization as root for container UID 10001 ownership')
|
||||
if (root / '.env').exists() or (root / 'runtime/connector/connector.key').exists():
|
||||
raise ValueError('Existing deployment detected; initialization will not overwrite it')
|
||||
settings = {k: v for k, v in dotenv_values(root / 'settings.env').items() if v is not None}
|
||||
required = ['HN_PUBLIC_URL', 'HN_FEISHU_API_BASE', 'HN_FEISHU_AUTHORIZE_URL']
|
||||
for field in required:
|
||||
value = settings.get(field, '')
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme != 'https' or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment:
|
||||
raise ValueError(f'{field} must be a valid HTTPS URL')
|
||||
if parsed.hostname.endswith('.invalid'):
|
||||
raise ValueError(f'Configure {field} before initializing')
|
||||
public = urlsplit(settings['HN_PUBLIC_URL'])
|
||||
if public.path not in ('', '/'):
|
||||
raise ValueError('HN_PUBLIC_URL must have no path prefix')
|
||||
if (public.port or 443) != int(settings.get('HN_HTTPS_PORT', '443')):
|
||||
raise ValueError('Public URL port must match HN_HTTPS_PORT')
|
||||
if settings.get('HN_FEISHU_ENABLED', 'true').lower() == 'true':
|
||||
if not all(settings.get(k) for k in ['HN_FEISHU_APP_ID', 'HN_FEISHU_APP_SECRET']):
|
||||
raise ValueError('Configure platform app ID and secret')
|
||||
if not re.fullmatch(r'[a-z0-9][a-z0-9_-]*', settings.get('COMPOSE_PROJECT_NAME', 'hainuo-relay')):
|
||||
raise ValueError('Invalid Compose project name')
|
||||
tls = root / 'runtime/tls'
|
||||
tls.mkdir(parents=True, exist_ok=True)
|
||||
cert_path, key_path = tls / 'server.pem', tls / 'server.key'
|
||||
if test_cert:
|
||||
if cert_path.exists() or key_path.exists():
|
||||
raise ValueError('Refusing to replace an existing TLS certificate')
|
||||
key = ed25519.Ed25519PrivateKey.generate()
|
||||
name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, public.hostname)])
|
||||
try:
|
||||
san = x509.IPAddress(ipaddress.ip_address(public.hostname))
|
||||
except ValueError:
|
||||
san = x509.DNSName(public.hostname)
|
||||
now = datetime.now(timezone.utc)
|
||||
cert = (x509.CertificateBuilder().subject_name(name).issuer_name(name).public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number()).not_valid_before(now-timedelta(minutes=5))
|
||||
.not_valid_after(now+timedelta(days=30))
|
||||
.add_extension(x509.SubjectAlternativeName([san]), critical=False).sign(key, None))
|
||||
key_path.write_bytes(pem_private(key))
|
||||
cert_path.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
|
||||
validate_tls(cert_path, key_path, public.hostname)
|
||||
ca_key, ca = issue_ca()
|
||||
signer = ed25519.Ed25519PrivateKey.generate()
|
||||
for folder in ['runtime/trust', 'runtime/connector', 'runtime/issuer']:
|
||||
(root / folder).mkdir(parents=True, exist_ok=False)
|
||||
(root / 'runtime/trust/ca.pem').write_bytes(ca.public_bytes(serialization.Encoding.PEM))
|
||||
(root / 'runtime/trust/connector.pub').write_bytes(signer.public_key().public_bytes(
|
||||
serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo))
|
||||
(root / 'runtime/connector/connector.key').write_bytes(pem_private(signer))
|
||||
(root / 'runtime/issuer/ca.key').write_bytes(pem_private(ca_key))
|
||||
redis_pass, db_pass = secrets.token_hex(32), secrets.token_hex(32)
|
||||
url = settings['HN_PUBLIC_URL'].rstrip('/')
|
||||
settings.update(HN_ENVIRONMENT='production', HN_PUBLIC_URL=url,
|
||||
HN_ROUTER_URL='wss://' + urlsplit(url).netloc + '/device/ws',
|
||||
HN_ADMIN_SECRET=secrets.token_hex(32), HN_INTERNAL_SECRET=secrets.token_hex(32),
|
||||
HN_REDIS_PASSWORD=redis_pass, HN_POSTGRES_PASSWORD=db_pass,
|
||||
HN_REDIS_URL=f'redis://:{redis_pass}@redis:6379/0',
|
||||
HN_DATABASE_URL=f'postgresql://hainuo:{db_pass}@postgres:5432/hainuo',
|
||||
HN_FEISHU_ENCRYPT_KEY='', HN_FEISHU_VERIFICATION_TOKEN='',
|
||||
HN_PROXY_ENABLED='false', HN_DEVICE_DOMAIN='')
|
||||
tmp = root / '.env.initializing'
|
||||
tmp.touch(mode=0o600, exist_ok=False)
|
||||
for key, value in settings.items():
|
||||
if '\n' in value or '\r' in value:
|
||||
raise ValueError('Multiline settings are unsupported')
|
||||
set_key(tmp, key, value, quote_mode='always')
|
||||
tmp.replace(root / '.env')
|
||||
for folder in ['runtime/connector', 'runtime/issuer']:
|
||||
path = root / folder
|
||||
path.chmod(0o700)
|
||||
if os.name != 'nt':
|
||||
os.chown(path, 10001, 10001)
|
||||
for file in path.iterdir():
|
||||
file.chmod(0o600)
|
||||
if os.name != 'nt':
|
||||
os.chown(file, 10001, 10001)
|
||||
(root / '.env').chmod(0o600)
|
||||
(root / 'settings.env').chmod(0o600)
|
||||
tls.chmod(0o700)
|
||||
key_path.chmod(0o600)
|
||||
cert_path.chmod(0o644)
|
||||
(root / 'runtime/trust').chmod(0o755)
|
||||
for path in (root / 'runtime/trust').iterdir():
|
||||
path.chmod(0o644)
|
||||
print('Initialized new deployment. No credentials printed. Preserve runtime/issuer/ca.key securely.')
|
||||
|
||||
|
||||
def validate_tls(cert_path, key_path, hostname):
|
||||
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
context.load_cert_chain(str(cert_path), str(key_path))
|
||||
cert = x509.load_pem_x509_certificate(cert_path.read_bytes())
|
||||
now = datetime.now(timezone.utc)
|
||||
if not cert.not_valid_before_utc <= now <= cert.not_valid_after_utc:
|
||||
raise ValueError('Gateway TLS certificate is not currently valid')
|
||||
san = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value
|
||||
try:
|
||||
address = ipaddress.ip_address(hostname)
|
||||
except ValueError:
|
||||
host = hostname.encode('idna').decode().lower()
|
||||
def matches(name):
|
||||
name = name.lower()
|
||||
return name == host or (name.startswith('*.') and name.count('*') == 1
|
||||
and len(host.split('.')) == len(name.split('.'))
|
||||
and host.split('.', 1)[1] == name[2:])
|
||||
valid = any(matches(x) for x in san.get_values_for_type(x509.DNSName))
|
||||
else:
|
||||
valid = address in san.get_values_for_type(x509.IPAddress)
|
||||
if not valid:
|
||||
raise ValueError('Gateway TLS certificate hostname mismatch')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--root', default='/work')
|
||||
parser.add_argument('--test-cert', action='store_true')
|
||||
args = parser.parse_args()
|
||||
initialize(args.root, args.test_cert)
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
case "$(uname -m)" in aarch64|arm64) ;; *) echo 'This dependency bundle requires Linux ARM64'; exit 1 ;; esac
|
||||
test -d images && test -d wheelhouse
|
||||
test -f OFFLINE_SHA256SUMS
|
||||
sha256sum -c OFFLINE_SHA256SUMS
|
||||
for archive in images/*.tar; do docker load -i "$archive"; done
|
||||
echo 'Offline images loaded. Next: build, configure settings.env, init, start.'
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Package current source with existing, checksum-verified Linux ARM64 dependencies.
|
||||
|
||||
Usage: python scripts/package_offline.py --dependencies /path/to/old-bundle --output /path/to/new.tar.gz
|
||||
The dependency directory must contain SHA256SUMS, images/, hainuo-relay/wheelhouse/.
|
||||
Only dependencies are reused; source in the old bundle is never packaged.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import tarfile
|
||||
import zipfile
|
||||
from email.parser import Parser
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def digest(path):
|
||||
with path.open('rb') as handle:
|
||||
return hashlib.file_digest(handle, 'sha256').hexdigest()
|
||||
|
||||
|
||||
def package(dependencies, output):
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
dependencies, output = Path(dependencies), Path(output)
|
||||
if output.exists():
|
||||
raise ValueError('Output already exists; refusing to overwrite')
|
||||
checks = {}
|
||||
for line in (dependencies/'SHA256SUMS').read_text(encoding='utf-8').splitlines():
|
||||
sha, rel = line.split(None, 1)
|
||||
checks[rel.strip().lstrip('*')] = sha
|
||||
assets = sorted(p for folder in [dependencies/'images', dependencies/'hainuo-relay/wheelhouse', dependencies/'licenses']
|
||||
for p in folder.rglob('*') if p.is_file())
|
||||
if not assets or not (dependencies/'licenses').is_dir():
|
||||
raise ValueError('Dependency bundle must include images, wheelhouse and license notices')
|
||||
wheel_versions = {}
|
||||
mapped = {}
|
||||
for path in assets:
|
||||
rel = path.relative_to(dependencies).as_posix()
|
||||
if rel not in checks or digest(path) != checks[rel]:
|
||||
raise ValueError('Dependency checksum mismatch: ' + rel)
|
||||
if path.suffix == '.whl':
|
||||
if not (path.name.endswith('none-any.whl') or 'aarch64' in path.name):
|
||||
raise ValueError('Unsupported wheel architecture: ' + path.name)
|
||||
with zipfile.ZipFile(path) as wheel:
|
||||
name = next(x for x in wheel.namelist() if x.endswith('.dist-info/METADATA'))
|
||||
meta = Parser().parsestr(wheel.read(name).decode('utf-8'))
|
||||
wheel_versions[meta['Name'].lower().replace('_','-')] = meta['Version']
|
||||
target = 'wheelhouse/' + path.name
|
||||
else:
|
||||
target = rel
|
||||
mapped[target] = path
|
||||
required = {}
|
||||
for line in (root/'requirements.lock').read_text(encoding='utf-8').splitlines():
|
||||
if line.strip() and not line.startswith('#'):
|
||||
name, version = line.split('==')
|
||||
required[name.lower().replace('_','-')] = version
|
||||
assert all(wheel_versions.get(n) == v for n,v in required.items()), 'Pinned wheel dependency mismatch'
|
||||
image_tars = list((dependencies/'images').glob('*.tar'))
|
||||
assert len(image_tars) == 4, 'Expected four base images'
|
||||
for path in image_tars:
|
||||
with tarfile.open(path) as image:
|
||||
manifest = json.load(image.extractfile('manifest.json'))
|
||||
for entry in manifest:
|
||||
config = json.load(image.extractfile(entry['Config']))
|
||||
assert config['architecture'] == 'arm64' and config['os'] == 'linux'
|
||||
for path in root.rglob('*'):
|
||||
rel = path.relative_to(root)
|
||||
if not path.is_file() or any(x in {'runtime','backups','wheelhouse','images','__pycache__','.pytest_cache','.git'} for x in rel.parts):
|
||||
continue
|
||||
if path.name in {'.env','settings.env'} or path.suffix == '.pyc':
|
||||
continue
|
||||
mapped[rel.as_posix()] = path
|
||||
manifest=''.join(f'{digest(path)} {name}\n' for name,path in sorted(mapped.items()))
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tarfile.open(output, 'w:gz', compresslevel=6) as tf:
|
||||
for name,path in sorted(mapped.items()):
|
||||
info=tf.gettarinfo(str(path), arcname='hainuo-relay-deployable/'+name)
|
||||
info.mode=0o755 if path.suffix=='.sh' else 0o644
|
||||
info.uid=info.gid=0
|
||||
info.uname=info.gname=''
|
||||
with path.open('rb') as handle:
|
||||
tf.addfile(info,handle)
|
||||
data=manifest.encode('utf-8')
|
||||
info=tarfile.TarInfo('hainuo-relay-deployable/OFFLINE_SHA256SUMS')
|
||||
info.size=len(data)
|
||||
info.mode=0o644
|
||||
tf.addfile(info,io.BytesIO(data))
|
||||
checksum=digest(output)
|
||||
Path(str(output)+'.sha256').write_text(checksum+' '+output.name+'\n',encoding='utf-8')
|
||||
print(json.dumps({'output':str(output),'bytes':output.stat().st_size,'sha256':checksum,
|
||||
'pinned_requirements':len(required),'verified_wheels':len(wheel_versions),'verified_images':len(image_tars)},indent=2))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--dependencies',required=True)
|
||||
parser.add_argument('--output',required=True)
|
||||
args=parser.parse_args()
|
||||
package(args.dependencies,args.output)
|
||||
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
# Execute on an INTERNET-CONNECTED LINUX ARM64 machine with Docker and Python 3.12.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
case "$(uname -m)" in aarch64|arm64) ;; *) echo 'Run on Linux ARM64'; exit 1 ;; esac
|
||||
python3 -c 'import sys; assert sys.version_info[:2] == (3,12)'
|
||||
mkdir -p images wheelhouse
|
||||
python3 -m pip download --only-binary=:all: --dest wheelhouse -r requirements.lock
|
||||
for spec in python:3.12-slim-bookworm postgres:17-alpine redis:8.2-alpine caddy:2-alpine; do
|
||||
docker pull --platform linux/arm64 "$spec"
|
||||
file=${spec//:/-}
|
||||
docker save -o "images/$file-linux-arm64.tar" "$spec"
|
||||
done
|
||||
find images wheelhouse -type f -print0 | sort -z | xargs -0 sha256sum > OFFLINE_SHA256SUMS
|
||||
echo 'Exact downloaded contents are recorded in OFFLINE_SHA256SUMS. Transfer the whole deployable directory.'
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
cd "$(dirname "$0")/.."
|
||||
action=${1:-help}
|
||||
mode=${2:-offline}
|
||||
args=(-f compose.yaml)
|
||||
case "$mode" in
|
||||
offline) args+=(-f compose.offline.yaml) ;;
|
||||
online) ;;
|
||||
*) echo 'Mode must be online or offline'; exit 1 ;;
|
||||
esac
|
||||
compose() { docker compose "${args[@]}" "$@"; }
|
||||
image=${HN_BUILD_IMAGE:-hainuo-relay:deploy-20261027}
|
||||
build() {
|
||||
if [[ -f .env ]]; then
|
||||
compose build connector
|
||||
return
|
||||
fi
|
||||
if [[ "$mode" == offline ]]; then
|
||||
test -d wheelhouse || { echo 'Missing wheelhouse; obtain the ARM64 offline bundle'; exit 1; }
|
||||
docker build --network none --pull=false -f Dockerfile.offline -t "$image" .
|
||||
else
|
||||
docker build -f Dockerfile -t "$image" .
|
||||
fi
|
||||
}
|
||||
case "$action" in
|
||||
build) build ;;
|
||||
init)
|
||||
test -f settings.env || { echo 'Copy .env.example to settings.env and configure it'; exit 1; }
|
||||
docker run --rm --user 0:0 --entrypoint python -e PYTHONPATH=/app -v "$PWD:/work" "$image" /work/scripts/initialize.py "${@:3}"
|
||||
;;
|
||||
check)
|
||||
test -f .env && test -f runtime/issuer/ca.key && test -f runtime/connector/connector.key
|
||||
test -f runtime/trust/ca.pem && test -f runtime/tls/server.pem && test -f runtime/tls/server.key
|
||||
compose config --quiet
|
||||
docker run --rm --user 0:0 --entrypoint python -e PYTHONPATH=/app -v "$PWD:/work:ro" "$image" /work/scripts/validate.py
|
||||
;;
|
||||
start)
|
||||
bash scripts/relayctl.sh check "$mode"
|
||||
compose up -d --no-build --pull never --wait --wait-timeout 180
|
||||
compose ps
|
||||
echo 'Containers ready. Now verify platform_connected and perform the documented business acceptance.'
|
||||
;;
|
||||
status) compose ps; compose exec -T connector python -m relay.cli status --url http://127.0.0.1:8000 ;;
|
||||
logs) compose logs --since 10m --tail 80 "${@:3}" ;;
|
||||
stop) compose stop ;;
|
||||
backup)
|
||||
stamp=$(date +%Y%m%d-%H%M%S)
|
||||
out="backups/$stamp"
|
||||
mkdir -p "$out"
|
||||
compose exec -T postgres pg_dump -U hainuo -d hainuo -Fc > "$out/metadata.dump.partial"
|
||||
mv "$out/metadata.dump.partial" "$out/metadata.dump"
|
||||
tar -czf "$out/config-and-keys.tar.gz" .env runtime
|
||||
cp compose*.yaml "$out/"
|
||||
compose images -q > "$out/image-ids.txt"
|
||||
(cd "$out" && sha256sum metadata.dump config-and-keys.tar.gz > SHA256SUMS)
|
||||
echo "Backup: $PWD/$out (contains private keys and secrets; restrict access). Not a restore verification."
|
||||
;;
|
||||
update)
|
||||
bash scripts/relayctl.sh backup "$mode"
|
||||
stamp=$(date +%Y%m%d-%H%M%S)
|
||||
current=$(compose images -q connector)
|
||||
test -n "$current"
|
||||
rollback="hainuo-relay:rollback-$stamp"
|
||||
docker image tag "$current" "$rollback"
|
||||
echo "Rollback image: $rollback (record this before proceeding)"
|
||||
build
|
||||
bash scripts/relayctl.sh start "$mode"
|
||||
;;
|
||||
rollback)
|
||||
previous=${3:?Provide the recorded rollback image tag}
|
||||
docker image inspect "$previous" >/dev/null
|
||||
HN_IMAGE="$previous" compose up -d --no-build --pull never --wait --wait-timeout 180
|
||||
echo 'Temporary rollback active. Set HN_IMAGE in .env to this tag before subsequent commands.'
|
||||
;;
|
||||
*) echo 'Usage: bash scripts/relayctl.sh {build|init|check|start|status|logs|stop|backup|update|rollback} {offline|online} [arguments]'; exit 1 ;;
|
||||
esac
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Read-only configuration/key consistency checks; never output secret values."""
|
||||
import os
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
from dotenv import dotenv_values
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from relay.config import Settings
|
||||
from relay.crypto import raw_public, load_private
|
||||
from initialize import validate_tls
|
||||
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--root',default='/work')
|
||||
root = Path(parser.parse_args().root)
|
||||
values = {k:v for k,v in dotenv_values(root/'.env').items() if v is not None}
|
||||
os.environ.update(values)
|
||||
Settings()
|
||||
public=urlsplit(values['HN_PUBLIC_URL'])
|
||||
validate_tls(root/'runtime/tls/server.pem',root/'runtime/tls/server.key',public.hostname)
|
||||
ca=x509.load_pem_x509_certificate((root/'runtime/trust/ca.pem').read_bytes())
|
||||
assert raw_public(load_private(root/'runtime/issuer/ca.key').public_key()) == raw_public(ca.public_key()), 'Issuer CA mismatch'
|
||||
pub=serialization.load_pem_public_key((root/'runtime/trust/connector.pub').read_bytes())
|
||||
assert raw_public(load_private(root/'runtime/connector/connector.key').public_key()) == raw_public(pub), 'Connector key mismatch'
|
||||
assert values['HN_ROUTER_URL']=='wss://'+public.netloc+'/device/ws', 'Router URL mismatch'
|
||||
print('Configuration, TLS hostname/expiry, CA and connector key pairs passed. Platform reachability not tested.')
|
||||
Reference in New Issue
Block a user