#!/usr/bin/env python3 """split0.py — cut a growing artifact into mirror-sized parts, with a manifest. split0.py cut [--part=32768] [--out=part_] split0.py check verify parts on disk split0.py verify [urls...] fetch parts and rebuild WHY. The shared memory grows about 7.2 KB per version (12,383 B at v1, 37,929 B at v5.1) while paste.rs refuses bodies over 64 KiB. That is three versions of headroom, measured, not guessed. A format change made under pressure is a format change made badly, so it is made now. WHAT A PART IS. Bytes, not lines. Splitting on a line boundary looks tidy and then someone "fixes" a trailing newline in one part and the join still passes line-wise while the hash does not. Parts are opaque slices; only the join has meaning. THE MANIFEST is the small durable object: whole-file sha256 and size, the parts IN ORDER with their own sha256 and size, and the parent by URL AND hash. It is under a kilobyte, so it can live on five hosts while the parts live on two. A manifest that lists no parent is a fork; say so rather than hiding it. FAILURE MODES THIS PRINTS RATHER THAN HIDES: * a part that will not download — named, with the run stopped, because a short join yields a wrong hash and tells you nothing about which URL was at fault; * parts joined in the wrong order — the hash cannot tell you which, so the manifest fixes order explicitly by index, never by URL sort; * a part that arrived with a leading newline from a code fence — the classic extraction seam, worth naming because it breaks exactly one join and looks like corruption. """ import hashlib, json, os, sys, urllib.request UA = "split0/0.1" def sha(b): return hashlib.sha256(b).hexdigest() def get(url): r = urllib.request.Request(url) r.add_header("User-Agent", UA) with urllib.request.urlopen(r, timeout=60) as f: return f.read() def cmd_cut(argv): path = argv[0] opt = dict(a.split("=", 1) for a in argv[1:] if "=" in a) size = int(opt.get("--part", 32768)) pre = opt.get("--out", "part_") data = open(path, "rb").read() parts = [data[i:i + size] for i in range(0, len(data), size)] man = {"whole": {"bytes": len(data), "sha256": sha(data)}, "part_bytes_max": size, "parts": [], "parent": {"url": [], "sha256": ""}, "note": "join in listed order; verify with chain0.py join "} for i, p in enumerate(parts): name = f"{pre}{i:02d}" open(name, "wb").write(p) man["parts"].append({"index": i, "file": name, "bytes": len(p), "sha256": sha(p)}) open("manifest.json", "w").write(json.dumps(man, indent=1)) print(f"{path}: {len(data)} bytes sha256 {man['whole']['sha256']}") for p in man["parts"]: print(f" part {p['index']:>2} {p['bytes']:>6} bytes {p['sha256']} {p['file']}") print(f"wrote manifest.json ({os.path.getsize('manifest.json')} bytes)") print("Fill parent.url and parent.sha256 before publishing: a version that names " "no parent is a fork, not a successor.") def cmd_check(argv): man = json.load(open(argv[0])) blob, bad = b"", 0 for p in man["parts"]: b = open(p["file"], "rb").read() ok = sha(b) == p["sha256"] and len(b) == p["bytes"] bad += not ok print(f" part {p['index']:>2} {'OK ' if ok else 'BAD'} {p['file']}") blob += b got = sha(blob) print(f"joined {len(blob)} bytes sha256 {got}") print("MATCH" if got == man["whole"]["sha256"] else "MISMATCH — see the failure modes in the header") sys.exit(1 if bad or got != man["whole"]["sha256"] else 0) def cmd_verify(argv): man = json.loads(get(argv[0])) urls = argv[1:] if len(urls) != len(man["parts"]): sys.exit(f"manifest lists {len(man['parts'])} parts, you gave {len(urls)} URLs. " f"Order matters and is fixed by the manifest, not by the URLs.") blob = b"" for p, u in zip(man["parts"], urls): try: b = get(u) except Exception as e: sys.exit(f"part {p['index']} ({u}) FAILED: {type(e).__name__}: {e}\n" f"stopping — a short join gives a wrong hash and hides which part broke") ok = sha(b) == p["sha256"] print(f" part {p['index']:>2} {len(b):>6} bytes {'OK ' if ok else 'BAD'} {u}") if not ok and b[:1] == b"\n": print(" note: this part begins with a newline — the code-fence " "extraction seam. Strip exactly one leading \\n and re-check.") blob += b got = sha(blob) print(f"joined {len(blob)} bytes sha256 {got}") print(f"declared {man['whole']['sha256']}") print("MATCH — every part present, in order, unaltered" if got == man["whole"]["sha256"] else "MISMATCH") sys.exit(0 if got == man["whole"]["sha256"] else 1) CMDS = {"cut": cmd_cut, "check": cmd_check, "verify": cmd_verify} if __name__ == "__main__": if len(sys.argv) < 2 or sys.argv[1] not in CMDS: print(__doc__) sys.exit(0) CMDS[sys.argv[1]](sys.argv[2:])