Add NPM certificate selection and read-only VDS certificate renewal sync
This commit is contained in:
283
homegate.py
283
homegate.py
@@ -5,6 +5,7 @@ import argparse
|
||||
import copy
|
||||
import fcntl
|
||||
import hashlib
|
||||
import io
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
@@ -21,11 +22,12 @@ from datetime import datetime, timedelta, timezone
|
||||
from urllib.parse import quote, urlsplit
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import uuid
|
||||
import zipfile
|
||||
|
||||
import requests
|
||||
import yaml
|
||||
from cryptography import x509
|
||||
from cryptography.exceptions import InvalidSignature
|
||||
from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
|
||||
@@ -113,7 +115,8 @@ def validate_config(value):
|
||||
for r in c["services"]:
|
||||
require(isinstance(r, dict), "Each service must be a mapping")
|
||||
require(not set(r) - {"hostname", "display_name", "upstream", "upstream_tls_verify", "upstream_tls_name",
|
||||
"upstream_ca_file", "websocket", "timeout_seconds", "max_body_mb", "enabled", "ssh"},
|
||||
"upstream_ca_file", "websocket", "timeout_seconds", "max_body_mb", "enabled", "ssh",
|
||||
"certificate_mode", "npm_certificate_id", "remote_certificate_id"},
|
||||
"Unknown service config key")
|
||||
if "display_name" in r:
|
||||
name = r["display_name"]
|
||||
@@ -126,6 +129,16 @@ def validate_config(value):
|
||||
"hostname must be a lowercase DNS name in domain_zones")
|
||||
require(host not in seen, f"Duplicate hostname: {host}")
|
||||
seen.add(host)
|
||||
mode = r.setdefault("certificate_mode", "homegate")
|
||||
require(mode in ("homegate", "npm", "remote_npm"), "Invalid certificate_mode")
|
||||
if mode == "npm" or "npm_certificate_id" in r:
|
||||
number(r.get("npm_certificate_id"), 1, 2147483647, "npm_certificate_id")
|
||||
if mode == "remote_npm":
|
||||
number(r.get("remote_certificate_id"), 1, 2147483647, "remote_certificate_id")
|
||||
else:
|
||||
require("remote_certificate_id" not in r, "remote_certificate_id requires remote_npm mode")
|
||||
require(mode != "homegate" or "npm_certificate_id" not in r,
|
||||
"npm_certificate_id requires an NPM certificate mode")
|
||||
u = urlsplit(r["upstream"])
|
||||
require(u.scheme in ("http", "https") and not u.username and not u.password
|
||||
and u.path in ("", "/") and not u.query and not u.fragment,
|
||||
@@ -151,6 +164,14 @@ def validate_config(value):
|
||||
"service.ssh accepts alias, host, port, user and enabled")
|
||||
boolean(s.get("enabled", True), "ssh.enabled")
|
||||
require(isinstance(c.get("ssh_hosts", []), list), "ssh_hosts must be a list")
|
||||
targets = {}
|
||||
for r in c["services"]:
|
||||
if r.get("npm_certificate_id"):
|
||||
target = r["npm_certificate_id"]
|
||||
source = (r["certificate_mode"], r.get("remote_certificate_id"))
|
||||
require(target not in targets or targets[target] == source,
|
||||
"A local certificate cannot have different renewal sources")
|
||||
targets[target] = source
|
||||
# Validate the complete inventory (including duplicate aliases) before any side effects.
|
||||
ssh_entries(c)
|
||||
return c
|
||||
@@ -455,6 +476,228 @@ class NPM(API):
|
||||
return result
|
||||
|
||||
|
||||
class RemoteNPM(API):
|
||||
"""Read-only production source. No renewal, upload, host or settings endpoints."""
|
||||
login = NPM.login
|
||||
|
||||
def call(self, method, path, **kwargs):
|
||||
require((method == "POST" and path == "/tokens") or
|
||||
(method == "GET" and path == "/nginx/certificates"),
|
||||
"Remote NPM is read-only: request refused")
|
||||
return super().call(method, path, **kwargs)
|
||||
|
||||
def download(self, cert_id):
|
||||
number(cert_id, 1, 2147483647, "remote_certificate_id")
|
||||
path = f"/nginx/certificates/{cert_id}/download"
|
||||
try:
|
||||
with self.session.get(self.base + path, timeout=(5, 30), allow_redirects=False, stream=True) as response:
|
||||
require(response.status_code == 200, f"Remote NPM: HTTP {response.status_code} downloading certificate")
|
||||
content = bytearray()
|
||||
for chunk in response.iter_content(65536):
|
||||
content.extend(chunk)
|
||||
require(len(content) <= 2 * 1024 * 1024, "Remote certificate archive is too large")
|
||||
return bytes(content)
|
||||
except requests.RequestException:
|
||||
raise Error("Remote NPM: certificate download connection or TLS error") from None
|
||||
|
||||
|
||||
def certificate_info(cert):
|
||||
"""Whitelist public fields: NPM responses can contain private keys and DNS tokens."""
|
||||
domains = cert.get("domain_names", [])
|
||||
expires = cert.get("expires_on")
|
||||
fingerprint = None
|
||||
pem = cert.get("meta", {}).get("certificate")
|
||||
if isinstance(pem, str) and "BEGIN CERTIFICATE" in pem:
|
||||
try:
|
||||
leaf = x509.load_pem_x509_certificate(pem.encode())
|
||||
domains = leaf.extensions.get_extension_for_class(x509.SubjectAlternativeName).value.get_values_for_type(x509.DNSName)
|
||||
expires = leaf.not_valid_after_utc.isoformat()
|
||||
fingerprint = leaf.fingerprint(hashes.SHA256()).hex()
|
||||
except (ValueError, x509.ExtensionNotFound, x509.DuplicateExtension):
|
||||
raise Error("NPM certificate has invalid PEM or missing DNS subject alternative names") from None
|
||||
return {"id": cert["id"], "name": cert.get("nice_name", ""), "provider": cert.get("provider"),
|
||||
"domains": domains, "expires": expires, "fingerprint": fingerprint,
|
||||
"modified": cert.get("modified_on")}
|
||||
|
||||
|
||||
def covers_domain(pattern, hostname):
|
||||
pattern = pattern.lower().rstrip(".")
|
||||
return pattern == hostname or (pattern.startswith("*.") and
|
||||
hostname.count(".") == pattern.count(".") and hostname.endswith(pattern[1:]))
|
||||
|
||||
|
||||
def check_certificate(info, hostnames):
|
||||
try:
|
||||
expires = datetime.fromisoformat(info["expires"].replace("Z", "+00:00"))
|
||||
if expires.tzinfo is None:
|
||||
expires = expires.replace(tzinfo=timezone.utc)
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
raise Error("NPM certificate has no valid expiration date") from None
|
||||
require(expires > now(), "NPM certificate has expired")
|
||||
for hostname in hostnames:
|
||||
require(any(covers_domain(d, hostname) for d in info["domains"]),
|
||||
f"NPM certificate does not cover {hostname}")
|
||||
return expires
|
||||
|
||||
|
||||
def remote_bundle(content, hostnames):
|
||||
"""Read bounded PEMs in memory, never extract untrusted ZIP paths to disk."""
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(content)) as archive:
|
||||
entries = archive.infolist()
|
||||
require(len(entries) <= 16 and sum(e.file_size for e in entries) <= 2 * 1024 * 1024,
|
||||
"Invalid remote certificate archive size")
|
||||
require(all(re.fullmatch(r"[a-zA-Z0-9_.-]+", e.filename) and not e.flag_bits & 1 for e in entries),
|
||||
"Invalid remote certificate archive entries")
|
||||
def read_one(pattern):
|
||||
matches = [e for e in entries if re.fullmatch(pattern, e.filename)]
|
||||
require(len(matches) == 1, "Remote archive must contain one fullchain and one private key")
|
||||
return archive.read(matches[0]).decode("ascii")
|
||||
fullchain = read_one(r"fullchain\d*\.pem")
|
||||
key_pem = read_one(r"privkey\d*\.pem")
|
||||
chain = x509.load_pem_x509_certificates(fullchain.encode())
|
||||
require(len(chain) >= 2, "Remote certificate must include the intermediate chain")
|
||||
leaf = chain[0]
|
||||
require(not leaf.extensions.get_extension_for_class(x509.BasicConstraints).value.ca,
|
||||
"Remote certificate must be a server certificate")
|
||||
key = serialization.load_pem_private_key(key_pem.encode(), password=None)
|
||||
public = lambda k: k.public_bytes(serialization.Encoding.DER, serialization.PublicFormat.SubjectPublicKeyInfo)
|
||||
require(public(key.public_key()) == public(leaf.public_key()), "Remote certificate and key do not match")
|
||||
require(leaf.not_valid_before_utc <= now(), "Remote certificate is not valid yet")
|
||||
for child, issuer in zip(chain, chain[1:]):
|
||||
child.verify_directly_issued_by(issuer)
|
||||
require(issuer.extensions.get_extension_for_class(x509.BasicConstraints).value.ca,
|
||||
"Invalid intermediate CA")
|
||||
require(issuer.not_valid_before_utc <= now() < issuer.not_valid_after_utc,
|
||||
"Remote intermediate certificate is not valid")
|
||||
info = certificate_info({"id": 0, "meta": {"certificate": fullchain}})
|
||||
check_certificate(info, hostnames)
|
||||
return {"cert": leaf.public_bytes(serialization.Encoding.PEM).decode(), "key": key_pem,
|
||||
"chain": "".join(c.public_bytes(serialization.Encoding.PEM).decode() for c in chain[1:]),
|
||||
"info": info}
|
||||
except (zipfile.BadZipFile, UnicodeError, ValueError, TypeError, InvalidSignature, NotImplementedError,
|
||||
UnsupportedAlgorithm, x509.ExtensionNotFound, x509.DuplicateExtension):
|
||||
raise Error("Invalid remote certificate archive, key or certificate chain") from None
|
||||
|
||||
|
||||
class CertificateSync:
|
||||
"""Copy renewed source certificates into the home NPM; retain last copy offline."""
|
||||
def __init__(self, npm, remote, settings, state, registry, routes):
|
||||
self.npm, self.remote, self.settings = npm, remote, settings or {}
|
||||
self.state, self.registry, self.routes = state, registry, routes
|
||||
self.sources = None
|
||||
self.source_error = None
|
||||
self.results = {}
|
||||
self.conflicts = set()
|
||||
targets = {}
|
||||
for route in routes:
|
||||
target_id = route.get("npm_certificate_id")
|
||||
if not target_id and route.get("certificate_mode") == "remote_npm":
|
||||
name = f'homegate:{registry["instance"]}:remote:{route["remote_certificate_id"]}'
|
||||
target_id = next((c["id"] for c in npm.certs if c.get("nice_name") == name), None)
|
||||
if target_id:
|
||||
source = (route["certificate_mode"], route.get("remote_certificate_id"))
|
||||
if target_id in targets and targets[target_id] != source:
|
||||
self.conflicts.add(target_id)
|
||||
targets[target_id] = source
|
||||
|
||||
def get_sources(self):
|
||||
if self.source_error:
|
||||
raise Error(self.source_error)
|
||||
if self.sources is None:
|
||||
try:
|
||||
require(self.remote is not None, "Configure the VDS NPM connection in Settings")
|
||||
self.remote.login(self.settings["remote_npm_identity"], self.settings["remote_npm_password"])
|
||||
self.sources = self.remote.call("GET", "/nginx/certificates")
|
||||
except (Error, KeyError):
|
||||
self.source_error = "Remote NPM unavailable; check its connection and certificate read permissions"
|
||||
raise Error(self.source_error) from None
|
||||
return self.sources
|
||||
|
||||
def select(self, route):
|
||||
target_id = route.get("npm_certificate_id")
|
||||
source_id = route.get("remote_certificate_id")
|
||||
name = f'homegate:{self.registry["instance"]}:remote:{source_id}'
|
||||
matches = [c for c in self.npm.certs if c["id"] == target_id] if target_id else [
|
||||
c for c in self.npm.certs if c.get("nice_name") == name]
|
||||
require(len(matches) <= 1, "Duplicate local NPM certificate")
|
||||
target = matches[0] if matches else None
|
||||
require(target is not None or not target_id, "Selected local NPM certificate no longer exists")
|
||||
require(not target or target["id"] not in self.conflicts,
|
||||
"Local certificate has conflicting renewal sources; use a separate copy")
|
||||
if route["certificate_mode"] == "npm":
|
||||
info = certificate_info(target)
|
||||
check_certificate(info, [route["hostname"]])
|
||||
signature = {"id": target["id"], "sha256": info["fingerprint"],
|
||||
"expires": info["expires"], "modified": info["modified"]}
|
||||
return target["id"], info, signature, None
|
||||
require(target is None or target["provider"] == "other",
|
||||
"VDS synchronization requires a custom certificate in the home NPM")
|
||||
cache_key = str(target["id"]) if target else name
|
||||
if cache_key in self.results:
|
||||
check_certificate(self.results[cache_key][1], [route["hostname"]])
|
||||
return self.results[cache_key]
|
||||
error = None
|
||||
# Validate every local proxy sharing this certificate before replacing it.
|
||||
hostnames = {r["hostname"] for r in self.routes if r.get("npm_certificate_id") == target_id and
|
||||
(target_id or r.get("remote_certificate_id") == source_id)}
|
||||
hostnames.update(d for host in self.npm.hosts if target and host.get("certificate_id") == target["id"]
|
||||
for d in host["domain_names"])
|
||||
records = self.registry.setdefault("certificate_copies", {})
|
||||
record = records.get(str(target["id"]), {}) if target else {}
|
||||
try:
|
||||
sources = [c for c in self.get_sources() if c["id"] == source_id]
|
||||
require(len(sources) == 1 and sources[0]["provider"] == "letsencrypt",
|
||||
"Select an existing Let's Encrypt certificate from VDS NPM")
|
||||
source = certificate_info(sources[0])
|
||||
check_certificate(source, hostnames)
|
||||
revision = {"url": self.remote.base, "id": source_id,
|
||||
"expires": source["expires"], "modified": source["modified"]}
|
||||
local = certificate_info(target) if target else None
|
||||
if (record.get("source") != revision or not local or
|
||||
local["fingerprint"] != record.get("fingerprint") or
|
||||
time.time() - record.get("checked_at", 0) >= 86400):
|
||||
bundle = remote_bundle(self.remote.download(source_id), hostnames)
|
||||
info = bundle["info"]
|
||||
if local and local["expires"]:
|
||||
# A source rolled back to an older certificate must not shorten local validity.
|
||||
try:
|
||||
old_expiry = check_certificate(local, hostnames)
|
||||
except Error:
|
||||
old_expiry = now()
|
||||
require(check_certificate(info, hostnames) >= old_expiry,
|
||||
"VDS certificate is older than the local copy")
|
||||
if not target:
|
||||
target = self.npm.call("POST", "/nginx/certificates", json={"provider": "other", "nice_name": name})
|
||||
self.npm.certs.append(target)
|
||||
digest = hashlib.sha256((bundle["cert"] + bundle["chain"]).encode()).hexdigest()
|
||||
if not local or local["fingerprint"] != info["fingerprint"] or record.get("sha256") != digest:
|
||||
self.npm.call("POST", f'/nginx/certificates/{target["id"]}/upload', files={
|
||||
"certificate": ("certificate.pem", bundle["cert"], "application/x-pem-file"),
|
||||
"certificate_key": ("key.pem", bundle["key"], "application/x-pem-file"),
|
||||
"intermediate_certificate": ("chain.pem", bundle["chain"], "application/x-pem-file"),
|
||||
})
|
||||
target.update(expires_on=info["expires"], domain_names=info["domains"],
|
||||
meta={"certificate": bundle["cert"]})
|
||||
record = {"source": revision, "fingerprint": info["fingerprint"], "sha256": digest,
|
||||
"checked_at": time.time(), "synced_at": time.time()}
|
||||
records[str(target["id"])] = record
|
||||
save_json(self.state / "registry.json", self.registry)
|
||||
except (Error, ValueError, KeyError, OSError) as exc:
|
||||
error = str(exc)
|
||||
LOG.warning("VDS certificate copy: %s", error)
|
||||
require(target is not None, error or "Local certificate copy is not available yet")
|
||||
info = certificate_info(target)
|
||||
check_certificate(info, [route["hostname"]])
|
||||
info["synced_at"] = record.get("synced_at")
|
||||
info["sync_error"] = error
|
||||
# The route records this fingerprint only after nginx accepted a reload.
|
||||
signature = {"id": target["id"], "sha256": record.get("sha256") or info["fingerprint"]}
|
||||
result = (target["id"], info, signature, error)
|
||||
self.results[str(target["id"])] = self.results[cache_key] = result
|
||||
return result
|
||||
|
||||
|
||||
class PiHole(API):
|
||||
def login(self, password):
|
||||
self.session.headers.pop("X-FTL-SID", None)
|
||||
@@ -525,8 +768,9 @@ def payload(config, route, cert_id, marker):
|
||||
"locations": [], "enabled": route["enabled"]}
|
||||
|
||||
|
||||
def reconcile(config, state, ca, registry, npm, dns):
|
||||
def reconcile(config, state, ca, registry, npm, dns, remote=None, settings=None):
|
||||
npm.inventory()
|
||||
certificates = CertificateSync(npm, remote, settings, state, registry, config["services"])
|
||||
dns_error = False
|
||||
if dns:
|
||||
try:
|
||||
@@ -547,18 +791,27 @@ def reconcile(config, state, ca, registry, npm, dns):
|
||||
existing = npm.find_host(host, marker)
|
||||
if dns and route["enabled"]:
|
||||
dns.check(host, config["proxy_ip"])
|
||||
bundle = ca.leaf(host)
|
||||
previous = registry["routes"].get(host, {})
|
||||
cert_id, changed, signature = npm.certificate(name, bundle, previous.get("uploaded"))
|
||||
cert_error = None
|
||||
if route.get("certificate_mode", "homegate") == "homegate":
|
||||
bundle = ca.leaf(host)
|
||||
cert_id, changed, signature = npm.certificate(name, bundle, previous.get("uploaded"))
|
||||
cert_info = None
|
||||
else:
|
||||
cert_id, cert_info, signature, cert_error = certificates.select(route)
|
||||
changed = signature != previous.get("uploaded")
|
||||
result = npm.host(existing, payload(config, route, cert_id, marker), force_reload=changed)
|
||||
# Save only AFTER nginx accepts the host/reload. Failed upload/reload is retried.
|
||||
registry["routes"][host] = {"host_id": result["id"], "uploaded": signature}
|
||||
registry["routes"][host] = {"host_id": result["id"], "uploaded": signature,
|
||||
"certificate": cert_info, "certificate_mode": route.get("certificate_mode", "homegate")}
|
||||
save_json(state / "registry.json", registry)
|
||||
# Publish DNS only once proxy and TLS are configured.
|
||||
if dns and route["enabled"]:
|
||||
dns.ensure(host, config["proxy_ip"])
|
||||
if changed or not existing:
|
||||
LOG.info("Configured https://%s -> %s", host, route["upstream"])
|
||||
if cert_error:
|
||||
failures.append(host)
|
||||
except (Error, ValueError, KeyError, OSError) as exc:
|
||||
failures.append(host)
|
||||
LOG.error("%s: %s", host, exc)
|
||||
@@ -583,6 +836,7 @@ def cycle(config, state, ca, registry):
|
||||
verify = str(ca_path)
|
||||
npm = NPM(settings["npm_url"] if settings is not None else os.environ.get("NPM_URL", ""), "NPM", verify)
|
||||
dns = None
|
||||
remote = None
|
||||
dns_error = False
|
||||
try:
|
||||
npm.login(settings["npm_identity"] if settings is not None else os.environ.get("NPM_IDENTITY", ""),
|
||||
@@ -596,12 +850,17 @@ def cycle(config, state, ca, registry):
|
||||
dns.close()
|
||||
dns = None
|
||||
dns_error = True
|
||||
reconcile(config, state, ca, registry, npm, dns)
|
||||
if settings and settings.get("remote_npm_url") and any(
|
||||
r.get("certificate_mode") == "remote_npm" for r in config["services"]):
|
||||
remote = RemoteNPM(settings["remote_npm_url"], "Remote NPM")
|
||||
reconcile(config, state, ca, registry, npm, dns, remote, settings)
|
||||
require(not dns_error, "Pi-hole login failed; proxy/certificate maintenance completed")
|
||||
finally:
|
||||
npm.close()
|
||||
if dns:
|
||||
dns.close()
|
||||
if remote:
|
||||
remote.close()
|
||||
|
||||
|
||||
def publish_panel_status(control, revision, config, state, ca, ok, error):
|
||||
@@ -609,11 +868,19 @@ def publish_panel_status(control, revision, config, state, ca, ok, error):
|
||||
control.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
previous = read_json(control / "status.json", {})
|
||||
certs = []
|
||||
registry = read_json(state / "registry.json", {"routes": {}})
|
||||
for route in config["services"]:
|
||||
mode = route.get("certificate_mode", "homegate")
|
||||
if mode != "homegate":
|
||||
saved = registry["routes"].get(route["hostname"], {})
|
||||
info = saved.get("certificate") if saved.get("certificate_mode") == mode else None
|
||||
if info and (not route.get("npm_certificate_id") or info["id"] == route["npm_certificate_id"]):
|
||||
certs.append({"hostname": route["hostname"], "mode": mode, **info})
|
||||
continue
|
||||
path = state / "issued" / (route["hostname"] + ".json")
|
||||
if path.exists():
|
||||
cert = x509.load_pem_x509_certificate(read_json(path)["cert"].encode())
|
||||
certs.append({"hostname": route["hostname"], "expires": cert.not_valid_after_utc.isoformat()})
|
||||
certs.append({"hostname": route["hostname"], "mode": mode, "expires": cert.not_valid_after_utc.isoformat()})
|
||||
save_json(control / "status.json", {
|
||||
"ok": ok, "time": time.time(), "max_age": max(180, config["interval_seconds"] * 3),
|
||||
"error": error, "applied_revision": revision if ok else previous.get("applied_revision"),
|
||||
|
||||
Reference in New Issue
Block a user