#!/usr/bin/env python3 """Deterministic digest-range sharding for the local patch-object store. The loose SHA-256 object store is authoritative. Shards are secondary deterministic MPOP v1 containers for the same bytes. They are local byte-integrity/recovery evidence only: not signing, authenticated backup custody, timestamp authority, public witness consensus, semantic truth, production disaster recovery, deployment proof, domain control, or real-world authorization. """ from __future__ import annotations from pathlib import Path from typing import Any, Iterable import argparse, copy, hashlib, json, os, shutil, tempfile ROOT=Path(__file__).resolve().parents[1] import sys sys.path.insert(0,str(ROOT/'tools')) import memory_patch_pack as pack MAX_OBJECTS_PER_SHARD=32 SHARD_DIR_REL='assets/patch-object-shards' SHARD_INDEX_DIR_REL='assets/data/patch-pack-shards' SUPER_REL='assets/data/memory-patch-shard-super-index.json' MIGRATION_REL='assets/data/memory-patch-shard-migration-results.json' CHECKPOINT_REL='assets/data/memory-patch-shard-checkpoints.json' SCRUB_REL='assets/data/memory-patch-shard-scrub-results.json' REBUILD_REL='assets/data/memory-patch-shard-rebuild-results.json' PHP_REL='assets/data/memory-patch-shard-php-results.json' MONOLITHIC_PACK_SHA256='f52de594ad40cc8293d83fb43c27fe1b1cf7586b1103deb8c1ddd7ba131e142a' MONOLITHIC_INDEX_FILE_SHA256='841d2ad6bc559fac3dbf38b21199eafce9958576d24ee4adb9d1bb148876ea1a' class ShardError(RuntimeError): pass def canonical_bytes(obj:Any)->bytes:return json.dumps(obj,ensure_ascii=False,sort_keys=True,separators=(',',':')).encode('utf-8') def sha_bytes(b:bytes)->str:return hashlib.sha256(b).hexdigest() def sha_file(p:Path)->str:return pack.sha_file(p) def load(p:Path)->dict[str,Any]:return json.loads(p.read_text('utf-8')) def write_json(p:Path,obj:Any)->None:p.parent.mkdir(parents=True,exist_ok=True);p.write_text(json.dumps(obj,ensure_ascii=False,indent=2)+'\n','utf-8') def shard_id(i:int)->str:return f'shard-{i:04d}' def shard_pack_rel(i:int)->str:return f'{SHARD_DIR_REL}/{shard_id(i)}.pack' def shard_index_rel(i:int)->str:return f'{SHARD_INDEX_DIR_REL}/{shard_id(i)}-index.json' def object_set_digest(objects:Iterable[dict[str,Any]])->str: material=''.join(f"{o['sha256']}\0{int(o.get('size_bytes',o.get('length',0)))}\n" for o in objects).encode() return sha_bytes(material) def chunks(objs:list[dict[str,Any]],n:int=MAX_OBJECTS_PER_SHARD)->list[list[dict[str,Any]]]: if n<=0:raise ShardError('bad_shard_bound') return [objs[i:i+n] for i in range(0,len(objs),n)] def build_subset_pack(root:Path,objs:list[dict[str,Any]])->tuple[bytes,list[dict[str,Any]],bytes,bytes]: count=len(objs); payload_offset=pack.HEADER_SIZE+count*pack.RECORD_SIZE; off=payload_offset; records=[]; payloads=[] for o in objs: dg=o['sha256']; p=root/pack.loose_rel(dg); data=p.read_bytes(); ln=len(data) if ln!=o['size_bytes'] or sha_bytes(data)!=dg:raise ShardError('loose_object_mismatch:'+dg) if ln>pack.MAX_OBJECT_SIZE or off+ln>pack.MAX_PACK_SIZE:raise ShardError('shard_bound') records.append({'sha256':dg,'offset':off,'length':ln});payloads.append(data);off+=ln header=pack.HEADER.pack(pack.MAGIC,pack.FORMAT_VERSION,pack.HEADER_SIZE,count,pack.RECORD_SIZE,payload_offset) idx=b''.join(pack.RECORD.pack(bytes.fromhex(r['sha256']),r['offset'],r['length']) for r in records) payload=b''.join(payloads); data=header+idx+payload parsed=pack.parse_pack_bytes(data,True) if parsed['object_count']!=count:raise ShardError('internal_parse') return data,records,idx,payload def build_shard_index(root:Path,ordinal:int,objs:list[dict[str,Any]])->tuple[bytes,dict[str,Any]]: if not objs or len(objs)>MAX_OBJECTS_PER_SHARD:raise ShardError('shard_object_count') data,records,index_bytes,payload=build_subset_pack(root,objs) first,last=objs[0]['sha256'],objs[-1]['sha256']; start=ordinal*MAX_OBJECTS_PER_SHARD;end=start+len(objs)-1 core={'schema':'memory-patch-shard-index/1','shard_format':'MPOP/1','shard_id':shard_id(ordinal),'ordinal':ordinal,'max_objects_per_shard':MAX_OBJECTS_PER_SHARD,'catalog_id':load(root/pack.CATALOG_REL)['catalog_id'],'catalog_digest_sha256':load(root/pack.CATALOG_REL)['catalog_digest_sha256'],'catalog_start_ordinal':start,'catalog_end_ordinal':end,'canonical_range':{'first_digest_inclusive':first,'last_digest_inclusive':last},'object_count':len(objs),'object_set_digest_sha256':object_set_digest(objs),'pack_path':shard_pack_rel(ordinal),'pack_size_bytes':len(data),'pack_sha256':sha_bytes(data),'binary_index_sha256':sha_bytes(index_bytes),'payload_region_sha256':sha_bytes(payload),'records':records} dg=sha_bytes(canonical_bytes(core));return data,{**core,'index_digest_sha256':dg,'index_id':'shard-index-'+dg[:20]} def build_all_shards(root:Path)->tuple[list[dict[str,Any]],dict[str,Any]]: cat,objs=pack.catalog_objects(root); all_chunks=chunks(objs); descriptors=[] sd=root/SHARD_DIR_REL;idp=root/SHARD_INDEX_DIR_REL;sd.mkdir(parents=True,exist_ok=True);idp.mkdir(parents=True,exist_ok=True) # remove only generated current shard files to avoid stale shard ambiguity for p in sd.glob('shard-*.pack'):p.unlink() for p in idp.glob('shard-*-index.json'):p.unlink() for i,sub in enumerate(all_chunks): data,idx=build_shard_index(root,i,sub);(root/idx['pack_path']).write_bytes(data);write_json(root/shard_index_rel(i),idx) descriptors.append({'shard_id':idx['shard_id'],'ordinal':i,'index_path':shard_index_rel(i),'pack_path':idx['pack_path'],'catalog_start_ordinal':idx['catalog_start_ordinal'],'catalog_end_ordinal':idx['catalog_end_ordinal'],'first_digest':idx['canonical_range']['first_digest_inclusive'],'last_digest':idx['canonical_range']['last_digest_inclusive'],'object_count':idx['object_count'],'object_set_digest_sha256':idx['object_set_digest_sha256'],'pack_sha256':idx['pack_sha256'],'pack_size_bytes':idx['pack_size_bytes'],'index_digest_sha256':idx['index_digest_sha256']}) mono=load(root/pack.INDEX_REL); pack.verify_index(root,mono) core={'schema':'memory-patch-shard-super-index/1','generation_rule':{'ordering':'full lowercase SHA-256 digest ascending','maximum_objects_per_shard':MAX_OBJECTS_PER_SHARD,'partition':'contiguous fixed-size slices of sorted authoritative catalog; final shard may contain fewer objects','range_semantics':'first/last digest inclusive and catalog ordinal interval inclusive'},'authority':'assets/patch-object-store/sha256','catalog_id':cat['catalog_id'],'catalog_digest_sha256':cat['catalog_digest_sha256'],'object_count':len(objs),'object_set_digest_sha256':object_set_digest(objs),'shard_count':len(descriptors),'monolithic_v1_binding':{'pack_path':mono['pack_path'],'pack_id':mono['pack_id'],'pack_sha256':mono['pack_sha256'],'index_digest_sha256':mono['index_digest_sha256'],'index_file_sha256':sha_file(root/pack.INDEX_REL)},'shards':descriptors} dg=sha_bytes(canonical_bytes(core));sup={**core,'super_index_digest_sha256':dg,'super_index_id':'shard-super-'+dg[:20],'truth_boundary':'Local deterministic sharding/migration byte-integrity evidence only. Loose SHA-256 objects remain authoritative; shards do not establish signing, authenticated custody, timestamps, semantic truth, public consensus, production disaster recovery, deployment, domain control, or real-world authorization.'} write_json(root/SUPER_REL,sup);return [load(root/shard_index_rel(i)) for i in range(len(descriptors))],sup def verify_shard_index(root:Path,idx:dict[str,Any],data:bytes|None=None)->dict[str,Any]: required=['schema','shard_format','shard_id','ordinal','max_objects_per_shard','catalog_id','catalog_digest_sha256','catalog_start_ordinal','catalog_end_ordinal','canonical_range','object_count','object_set_digest_sha256','pack_path','pack_size_bytes','pack_sha256','binary_index_sha256','payload_region_sha256','records','index_digest_sha256','index_id'] if any(k not in idx for k in required):raise ShardError('shard_index_missing_field') if idx['schema']!='memory-patch-shard-index/1' or idx['shard_format']!='MPOP/1':raise ShardError('shard_index_schema') if idx['max_objects_per_shard']!=MAX_OBJECTS_PER_SHARD:raise ShardError('shard_bound_metadata') if idx['shard_id']!=shard_id(idx['ordinal']):raise ShardError('swapped_shard_id') b=data if data is not None else (root/idx['pack_path']).read_bytes();parsed=pack.parse_pack_bytes(b,True) for k in ('object_count','pack_size_bytes','pack_sha256','binary_index_sha256','payload_region_sha256','records'): if idx[k]!=parsed[k]:raise ShardError('shard_pack_binding:'+k) if idx['object_count']<1 or idx['object_count']>MAX_OBJECTS_PER_SHARD:raise ShardError('shard_object_count') if idx['catalog_end_ordinal']-idx['catalog_start_ordinal']+1!=idx['object_count']:raise ShardError('ordinal_span') if idx['records'][0]['sha256']!=idx['canonical_range']['first_digest_inclusive'] or idx['records'][-1]['sha256']!=idx['canonical_range']['last_digest_inclusive']:raise ShardError('wrong_first_last_range') objs=[{'sha256':r['sha256'],'size_bytes':r['length']} for r in idx['records']] if object_set_digest(objs)!=idx['object_set_digest_sha256']:raise ShardError('shard_object_set_digest') core={k:idx[k] for k in required if k not in ('index_digest_sha256','index_id')};dg=sha_bytes(canonical_bytes(core)) if idx['index_digest_sha256']!=dg or idx['index_id']!='shard-index-'+dg[:20]:raise ShardError('shard_index_digest') return {'pass':True,'shard_id':idx['shard_id'],'object_count':idx['object_count'],'pack_sha256':parsed['pack_sha256'],'index_digest_sha256':dg,'first_digest':idx['canonical_range']['first_digest_inclusive'],'last_digest':idx['canonical_range']['last_digest_inclusive']} def verify_super(root:Path,super_index:dict[str,Any]|None=None,shard_indexes:dict[str,dict[str,Any]]|None=None,shard_bytes:dict[str,bytes]|None=None,verify_monolithic:bool=True)->dict[str,Any]: sup=copy.deepcopy(super_index if super_index is not None else load(root/SUPER_REL));cat,objs=pack.catalog_objects(root);expected_digests=[o['sha256'] for o in objs] if sup.get('schema')!='memory-patch-shard-super-index/1':raise ShardError('super_schema') rule=sup.get('generation_rule') or {} if rule.get('maximum_objects_per_shard')!=MAX_OBJECTS_PER_SHARD:raise ShardError('super_bound') if sup.get('catalog_id')!=cat['catalog_id'] or sup.get('catalog_digest_sha256')!=cat['catalog_digest_sha256']:raise ShardError('super_catalog_binding') if sup.get('object_count')!=len(objs) or sup.get('object_set_digest_sha256')!=object_set_digest(objs):raise ShardError('super_object_set') descs=sup.get('shards') or [] if sup.get('shard_count')!=len(descs) or len(descs)!=len(chunks(objs)):raise ShardError('shard_count') all_d=[];verified=[];last_end=-1;last_digest='' for i,d in enumerate(descs): if d.get('ordinal')!=i or d.get('shard_id')!=shard_id(i):raise ShardError('swapped_shard_id') if d.get('catalog_start_ordinal')!=last_end+1:raise ShardError('range_gap_or_overlap') if d.get('catalog_end_ordinal',-1)dict[str,Any]: if len(dg)!=64 or any(c not in '0123456789abcdef' for c in dg):raise ShardError('bad_digest') hits=[d for d in super_index['shards'] if d['first_digest']<=dg<=d['last_digest']] if len(hits)!=1:raise ShardError('digest_range_resolution:'+str(len(hits))) return hits[0] def sparse_payload(root:Path,dg:str,super_index:dict[str,Any]|None=None)->bytes: sup=super_index or load(root/SUPER_REL);d=resolve_shard(sup,dg);idx=load(root/d['index_path']);data=(root/d['pack_path']).read_bytes();verify_shard_index(root,idx,data);hits=[r for r in idx['records'] if r['sha256']==dg] if len(hits)!=1:raise ShardError('digest_not_in_resolved_shard') return pack.payload_for(data,hits[0]) def extract(root:Path,destination:Path,digests:Iterable[str]|None=None)->dict[str,Any]: sup=load(root/SUPER_REL);verify_super(root,sup);dest=Path(destination) if dest.exists() and any(dest.iterdir()):raise ShardError('destination_not_fresh') dest.mkdir(parents=True,exist_ok=True) requested=None if digests is None else list(digests) if requested is not None and len(requested)!=len(set(requested)):raise ShardError('duplicate_sparse_digest') target=[o['sha256'] for o in load(root/pack.CATALOG_REL)['objects']] if requested is None else sorted(requested) valid={o['sha256'] for o in load(root/pack.CATALOG_REL)['objects']} if any(d not in valid for d in target):raise ShardError('unknown_sparse_digest') touched=set() for dg in target: d=resolve_shard(sup,dg);touched.add(d['shard_id']);payload=sparse_payload(root,dg,sup);p=dest/pack.extracted_rel(dg);p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(payload) return {'pass':True,'object_count':len(target),'shards_touched':sorted(touched),'sparse':requested is not None} def verify_extraction(root:Path,dest:Path,digests:Iterable[str]|None=None)->dict[str,Any]: targets=[o['sha256'] for o in load(root/pack.CATALOG_REL)['objects']] if digests is None else sorted(digests);bad=[] for dg in targets: p=dest/pack.extracted_rel(dg);loose=root/pack.loose_rel(dg) if not p.is_file() or p.read_bytes()!=loose.read_bytes():bad.append(dg) return {'pass':not bad,'object_count':len(targets),'mismatches':bad} def merkle_root(items:list[dict[str,Any]])->tuple[str,list[str]]: if not items:raise ShardError('empty_merkle') leaves=[sha_bytes(b'\x00'+canonical_bytes(x)) for x in items];level=[bytes.fromhex(x) for x in leaves] while len(level)>1: if len(level)%2:level.append(level[-1]) level=[hashlib.sha256(b'\x01'+level[i]+level[i+1]).digest() for i in range(0,len(level),2)] return level[0].hex(),leaves def build_checkpoints(root:Path)->dict[str,Any]: sup=load(root/SUPER_REL);verify_super(root,sup);per=[] for d in sup['shards']: idx=load(root/d['index_path']);items=[{'sha256':r['sha256'],'size_bytes':r['length']} for r in idx['records']];rootdg,leaves=merkle_root(items);per.append({'shard_id':d['shard_id'],'object_count':len(items),'first_digest':d['first_digest'],'last_digest':d['last_digest'],'pack_sha256':d['pack_sha256'],'index_digest_sha256':d['index_digest_sha256'],'merkle_root_sha256':rootdg,'leaf_digests_sha256':leaves}) super_items=[{'shard_id':d['shard_id'],'ordinal':d['ordinal'],'first_digest':d['first_digest'],'last_digest':d['last_digest'],'object_count':d['object_count'],'pack_sha256':d['pack_sha256'],'index_digest_sha256':d['index_digest_sha256']} for d in sup['shards']];sroot,sleaves=merkle_root(super_items) core={'schema':'memory-patch-shard-checkpoints/1','super_index_id':sup['super_index_id'],'object_count':sup['object_count'],'per_shard':per,'super_index_checkpoint':{'leaf_count':len(super_items),'merkle_root_sha256':sroot,'leaf_digests_sha256':sleaves}} dg=sha_bytes(canonical_bytes(core));out={**core,'checkpoint_digest_sha256':dg,'checkpoint_id':'shard-checkpoint-'+dg[:20]};write_json(root/CHECKPOINT_REL,out);return out def verify_checkpoints(root:Path,cp:dict[str,Any]|None=None)->dict[str,Any]: got=cp or load(root/CHECKPOINT_REL);sup=load(root/SUPER_REL);expected=build_checkpoints_in_memory(root,sup) if got!=expected:raise ShardError('checkpoint_substitution') return {'pass':True,'checkpoint_id':got['checkpoint_id'],'checkpoint_digest_sha256':got['checkpoint_digest_sha256'],'super_merkle_root_sha256':got['super_index_checkpoint']['merkle_root_sha256'],'shard_count':len(got['per_shard'])} def build_checkpoints_in_memory(root:Path,sup:dict[str,Any])->dict[str,Any]: verify_super(root,sup);per=[] for d in sup['shards']: idx=load(root/d['index_path']);items=[{'sha256':r['sha256'],'size_bytes':r['length']} for r in idx['records']];rootdg,leaves=merkle_root(items);per.append({'shard_id':d['shard_id'],'object_count':len(items),'first_digest':d['first_digest'],'last_digest':d['last_digest'],'pack_sha256':d['pack_sha256'],'index_digest_sha256':d['index_digest_sha256'],'merkle_root_sha256':rootdg,'leaf_digests_sha256':leaves}) items=[{'shard_id':d['shard_id'],'ordinal':d['ordinal'],'first_digest':d['first_digest'],'last_digest':d['last_digest'],'object_count':d['object_count'],'pack_sha256':d['pack_sha256'],'index_digest_sha256':d['index_digest_sha256']} for d in sup['shards']];sroot,sleaves=merkle_root(items) core={'schema':'memory-patch-shard-checkpoints/1','super_index_id':sup['super_index_id'],'object_count':sup['object_count'],'per_shard':per,'super_index_checkpoint':{'leaf_count':len(items),'merkle_root_sha256':sroot,'leaf_digests_sha256':sleaves}};dg=sha_bytes(canonical_bytes(core));return {**core,'checkpoint_digest_sha256':dg,'checkpoint_id':'shard-checkpoint-'+dg[:20]} def materialize_bundles_from_shards(root:Path)->dict[str,Any]: refs=load(root/pack.REFS_REL);bundles=[] for b in refs['bundles']: with tempfile.TemporaryDirectory() as td: out=Path(td);seen=set() for ref in b['references']: path=Path(ref['path']) if path.is_absolute() or '..' in path.parts or '\\' in ref['path']:raise ShardError('unsafe_bundle_path') dg=ref['object_sha256'];payload=sparse_payload(root,dg);p=out/path;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(payload);seen.add(path.as_posix()) lines=[f"{p.relative_to(out).as_posix()}\0{sha_file(p)}\n" for p in sorted(x for x in out.rglob('*') if x.is_file())];tree=sha_bytes(''.join(lines).encode()) import memory_patch_object_store as osmod payload_root=None for manifest,pr in osmod.BUNDLES: if manifest==b['bundle_manifest']:payload_root=pr;break if payload_root is None:raise ShardError('unknown_bundle_manifest') old=[f"{p.relative_to(root/payload_root).as_posix()}\0{sha_file(p)}\n" for p in sorted(x for x in (root/payload_root).rglob('*') if x.is_file())];oldtree=sha_bytes(''.join(old).encode()) bundles.append({'bundle_id':b['bundle_id'],'file_count':len(seen),'shard_tree_digest_sha256':tree,'preserved_tree_digest_sha256':oldtree,'byte_for_byte_parity':tree==oldtree}) core={'schema':'memory-patch-shard-bundle-materialization/1','super_index_id':load(root/SUPER_REL)['super_index_id'],'bundles':bundles};dg=sha_bytes(canonical_bytes(core));return {**core,'materialization_digest_sha256':dg,'pass':all(b['byte_for_byte_parity'] for b in bundles)} def build_migration(root:Path)->dict[str,Any]: supv=verify_super(root);midx=load(root/pack.INDEX_REL);cat=load(root/pack.CATALOG_REL);mat=materialize_bundles_from_shards(root) mono=[{'sha256':r['sha256'],'size_bytes':r['length']} for r in midx['records']];loose=[{'sha256':o['sha256'],'size_bytes':o['size_bytes']} for o in cat['objects']] shard=[] for d in load(root/SUPER_REL)['shards']: idx=load(root/d['index_path']);shard.extend({'sha256':r['sha256'],'size_bytes':r['length']} for r in idx['records']) refs=load(root/pack.REFS_REL);protected=sum(1 for b in refs['bundles'] for r in b['references'] if r.get('protected_anchor')) core={'schema':'memory-patch-shard-migration-results/1','monolithic_pack_id':midx['pack_id'],'super_index_id':load(root/SUPER_REL)['super_index_id'],'object_count':len(loose),'loose_object_set_digest_sha256':object_set_digest(loose),'monolithic_object_set_digest_sha256':object_set_digest(mono),'shard_object_set_digest_sha256':object_set_digest(shard),'object_sets_equal':loose==mono==shard,'bundle_materialization':mat,'bundle_path_binding_reference_count':sum(b['reference_count'] for b in refs['bundles']),'retention_digest_sha256':load(root/pack.RETENTION_REL)['retention_digest_sha256'],'protected_anchor_reference_count':protected,'protected_anchor_boundary_unchanged':protected==0,'monolithic_pack_bytes_preserved':sha_file(root/pack.PACK_REL)==MONOLITHIC_PACK_SHA256,'monolithic_index_bytes_preserved':sha_file(root/pack.INDEX_REL)==MONOLITHIC_INDEX_FILE_SHA256} dg=sha_bytes(canonical_bytes(core));out={**core,'migration_digest_sha256':dg,'migration_id':'shard-migration-'+dg[:20],'pass':core['object_sets_equal'] and mat['pass'] and core['protected_anchor_boundary_unchanged'] and core['monolithic_pack_bytes_preserved'] and core['monolithic_index_bytes_preserved'],'truth_boundary':'Migration proof establishes local object/payload/path/retention byte-state equality only; repacking does not establish authorship, authenticated custody, semantic truth, deployment, or external authority.'};write_json(root/MIGRATION_REL,out);return out def verify_authoritative_loose(root:Path)->dict[str,Any]: cat,objs=pack.catalog_objects(root);status=pack.loose_status(root) if not status['pass'] or status['healthy_object_count']!=len(objs):raise ShardError('authoritative_loose_store_not_verified') return {'pass':True,'catalog_id':cat['catalog_id'],'object_count':len(objs),'object_set_digest_sha256':object_set_digest(objs)} def expected_shard(root:Path,shard_name:str)->tuple[bytes,dict[str,Any]]: sup=load(root/SUPER_REL);hits=[d for d in sup['shards'] if d['shard_id']==shard_name] if len(hits)!=1:raise ShardError('unknown_shard') d=hits[0];_,objs=pack.catalog_objects(root);sub=objs[d['catalog_start_ordinal']:d['catalog_end_ordinal']+1];data,idx=build_shard_index(root,d['ordinal'],sub) if data and sha_bytes(data)!=d['pack_sha256']:raise ShardError('rebuild_expected_digest_disagreement') if idx['index_digest_sha256']!=d['index_digest_sha256']:raise ShardError('rebuild_expected_index_disagreement') return data,idx def rebuild_shard_from_loose(root:Path,shard_root:Path,shard_name:str,simulate_interrupt:bool=False)->dict[str,Any]: loose=verify_authoritative_loose(root);sup=load(root/SUPER_REL);verify_super(root,sup);d=next((x for x in sup['shards'] if x['shard_id']==shard_name),None) if d is None:raise ShardError('unknown_shard') target=shard_root/Path(d['pack_path']).name;state='missing' if target.is_file():state='healthy' if sha_file(target)==d['pack_sha256'] else 'corrupt' if state=='healthy':return {'pass':True,'shard_id':shard_name,'prior_state':'healthy','action':'none','authoritative_loose_verified':True,'automatic_delete_count':0} data,_=expected_shard(root,shard_name);target.parent.mkdir(parents=True,exist_ok=True);tmp=target.with_name('.'+target.name+'.rebuild.tmp');tmp.write_bytes(data) if sha_file(tmp)!=d['pack_sha256']:tmp.unlink(missing_ok=True);raise ShardError('staged_shard_verify') if simulate_interrupt:tmp.unlink(missing_ok=True);raise ShardError('simulated_shard_rebuild_interruption') os.replace(tmp,target) if sha_file(target)!=d['pack_sha256']:raise ShardError('final_shard_verify') return {'pass':True,'shard_id':shard_name,'prior_state':state,'action':'reconstructed_from_fully_verified_authoritative_loose_store','authoritative_loose_verified':loose['pass'],'automatic_delete_count':0,'protected_anchor_mutation_count':0} def build_rebuild_fixture(root:Path)->dict[str,Any]: sup=load(root/SUPER_REL);first=sup['shards'][0]['shard_id'];mid=sup['shards'][len(sup['shards'])//2]['shard_id'];last=sup['shards'][-1]['shard_id'] def copy_shards(base:Path): base.mkdir(parents=True,exist_ok=True) for d in sup['shards']:shutil.copy2(root/d['pack_path'],base/Path(d['pack_path']).name) with tempfile.TemporaryDirectory() as td: base=Path(td)/'shards';copy_shards(base);(base/f'{first}.pack').unlink();a=rebuild_shard_from_loose(root,base,first);missing_ok=sha_file(base/f'{first}.pack')==next(d['pack_sha256'] for d in sup['shards'] if d['shard_id']==first) with tempfile.TemporaryDirectory() as td: base=Path(td)/'shards';copy_shards(base);(base/f'{mid}.pack').write_bytes(b'corrupt-shard');b=rebuild_shard_from_loose(root,base,mid);corrupt_ok=sha_file(base/f'{mid}.pack')==next(d['pack_sha256'] for d in sup['shards'] if d['shard_id']==mid) with tempfile.TemporaryDirectory() as td: base=Path(td)/'shards';copy_shards(base);(base/f'{last}.pack').unlink();interrupted=False try:rebuild_shard_from_loose(root,base,last,True) except ShardError as e:interrupted=str(e)=='simulated_shard_rebuild_interruption' interruption_ok=interrupted and not (base/f'{last}.pack').exists() core={'schema':'memory-patch-shard-rebuild-results/1','super_index_id':sup['super_index_id'],'fixtures':[{'shard_id':first,'prior_state':'missing','pass':a['pass'] and missing_ok},{'shard_id':mid,'prior_state':'corrupt','pass':b['pass'] and corrupt_ok},{'shard_id':last,'prior_state':'missing_interrupted','pass':interruption_ok}],'authoritative_source':'assets/patch-object-store/sha256','authoritative_loose_store_verified':True,'automatic_delete_count':0,'protected_anchor_mutation_count':0} dg=sha_bytes(canonical_bytes(core));out={**core,'rebuild_digest_sha256':dg,'rebuild_id':'shard-rebuild-'+dg[:20],'pass':all(x['pass'] for x in core['fixtures']),'truth_boundary':'Isolated local shard rebuild only. It reconstructs secondary shard containers from a fully verified authoritative loose store and never reconstructs authoritative loose objects from shards.'};write_json(root/REBUILD_REL,out);return out def build_scrub(root:Path)->dict[str,Any]: sup=verify_super(root);migration=build_migration(root);cp=build_checkpoints_in_memory(root,load(root/SUPER_REL));loose=verify_authoritative_loose(root) with tempfile.TemporaryDirectory() as td: dest=Path(td)/'extract';extract(root,dest);full=verify_extraction(root,dest) core={'schema':'memory-patch-shard-scrub-results/1','super_index_id':sup['super_index_id'],'loose_store':loose,'shard_union':sup,'migration':migration,'checkpoints':{'checkpoint_id':cp['checkpoint_id'],'checkpoint_digest_sha256':cp['checkpoint_digest_sha256'],'super_merkle_root_sha256':cp['super_index_checkpoint']['merkle_root_sha256']},'full_extraction':full,'automatic_delete_count':0,'protected_anchor_mutation_count':0} dg=sha_bytes(canonical_bytes(core));out={**core,'scrub_digest_sha256':dg,'scrub_id':'shard-scrub-'+dg[:20],'pass':all([sup['pass'],migration['pass'],full['pass']]),'truth_boundary':'Local cross-representation byte scrub only; no signing, external custody, public witness, semantic truth, production recovery, deployment, domain control, or real-world authorization is established.'};write_json(root/SCRUB_REL,out);return out def build_all(root:Path)->dict[str,Any]: # Byte-for-byte predecessor preservation before generation. if sha_file(root/pack.PACK_REL)!=MONOLITHIC_PACK_SHA256 or sha_file(root/pack.INDEX_REL)!=MONOLITHIC_INDEX_FILE_SHA256:raise ShardError('monolithic_predecessor_changed') _,sup=build_all_shards(root);migration=build_migration(root);cp=build_checkpoints(root);scrub=build_scrub(root);rebuild=build_rebuild_fixture(root) return {'super_index':sup,'migration':migration,'checkpoints':cp,'scrub':scrub,'rebuild':rebuild} def main(): ap=argparse.ArgumentParser();ap.add_argument('--root',default=str(ROOT));ap.add_argument('--build',action='store_true');ap.add_argument('--verify-only',action='store_true');ap.add_argument('--extract');ap.add_argument('--digest',action='append');args=ap.parse_args();root=Path(args.root).resolve() if args.build:out=build_all(root) elif args.extract:out=extract(root,Path(args.extract),args.digest) else:out={'super_index':verify_super(root),'checkpoints':verify_checkpoints(root),'migration':build_migration(root),'scrub':build_scrub(root)} print(json.dumps(out,ensure_ascii=False)) if __name__=='__main__':main()