#!/usr/bin/env python3
"""Deterministic bounded packfile for the first-party SHA-256 patch-object store.

The loose SHA-256 object store remains authoritative. The pack is a local deterministic
byte-integrity and recovery aid only. It is not signing, an authenticated backup,
public witness consensus, production disaster recovery, deployment proof, domain
control, semantic truth, or real-world authorization.
"""
from __future__ import annotations
from pathlib import Path
from typing import Any, Iterable
import argparse, hashlib, json, os, shutil, struct, tempfile

ROOT = Path(__file__).resolve().parents[1]
MAGIC=b'MPOP'
FORMAT_VERSION=1
HEADER=struct.Struct('>4sHHIIQ')  # magic, version, header_size, count, record_size, payload_offset
RECORD=struct.Struct('>32sQQ')    # sha256 bytes, absolute payload offset, payload length
HEADER_SIZE=HEADER.size           # 24
RECORD_SIZE=RECORD.size           # 48
MAX_OBJECTS=65535
MAX_OBJECT_SIZE=1<<30             # 1 GiB per object, local safety bound
MAX_PACK_SIZE=8<<30               # 8 GiB total, local safety bound
PACK_REL='assets/patch-object-pack/memory-patch-objects-v1.pack'
INDEX_REL='assets/data/memory-patch-pack-index.json'
SCRUB_REL='assets/data/memory-patch-pack-scrub-results.json'
REPAIR_REL='assets/data/memory-patch-pack-repair-results.json'
CATALOG_REL='assets/data/memory-patch-object-catalog.json'
REFS_REL='assets/data/memory-patch-object-references.json'
RETENTION_REL='assets/data/memory-patch-object-retention.json'
CHECKPOINT_REL='assets/data/memory-patch-object-checkpoint.json'
LOOSE_REL='assets/patch-object-store/sha256'
PROTECTED={'.uai/totem.uai','.uai/taboo.uai','.uai/talisman.uai'}

class PackError(RuntimeError): pass

def canonical_bytes(obj:Any)->bytes:
    return json.dumps(obj,ensure_ascii=False,sort_keys=True,separators=(',',':')).encode('utf-8')
def sha_bytes(data:bytes)->str:return hashlib.sha256(data).hexdigest()
def sha_file(p:Path)->str:
    h=hashlib.sha256()
    with p.open('rb') as f:
        for c in iter(lambda:f.read(1024*1024),b''):h.update(c)
    return h.hexdigest()
def load(p:Path)->dict[str,Any]:return json.loads(p.read_text('utf-8'))
def loose_rel(dg:str)->str:
    if len(dg)!=64 or any(c not in '0123456789abcdef' for c in dg):raise PackError('bad_digest')
    return f'{LOOSE_REL}/{dg[:2]}/{dg}'
def extracted_rel(dg:str)->str:return f'sha256/{dg[:2]}/{dg}'

def catalog_objects(root:Path)->tuple[dict[str,Any],list[dict[str,Any]]]:
    cat=load(root/CATALOG_REL); objs=cat.get('objects') or []
    if len(objs)>MAX_OBJECTS:raise PackError('object_count_bound')
    if objs!=sorted(objs,key=lambda x:x['sha256']):raise PackError('catalog_unsorted')
    seen=set()
    for o in objs:
        dg=o['sha256']; sz=o['size_bytes']
        if dg in seen:raise PackError('catalog_duplicate_digest')
        seen.add(dg)
        if sz<0 or sz>MAX_OBJECT_SIZE:raise PackError('object_size_bound')
        p=root/loose_rel(dg)
        if not p.is_file():raise PackError('missing_loose_object:'+dg)
        if p.stat().st_size!=sz:raise PackError('loose_size:'+dg)
        if sha_file(p)!=dg:raise PackError('loose_digest:'+dg)
    return cat,objs

def build_pack_bytes(root:Path)->tuple[bytes,list[dict[str,Any]],dict[str,Any]]:
    cat,objs=catalog_objects(root)
    count=len(objs); payload_offset=HEADER_SIZE+count*RECORD_SIZE
    if payload_offset>MAX_PACK_SIZE:raise PackError('index_size_bound')
    records=[]; chunks=[]; off=payload_offset
    for o in objs:
        dg=o['sha256']; data=(root/loose_rel(dg)).read_bytes(); ln=len(data)
        if off>0xffffffffffffffff or ln>0xffffffffffffffff or off+ln>0xffffffffffffffff:raise PackError('uint64_overflow')
        if off+ln>MAX_PACK_SIZE:raise PackError('pack_size_bound')
        records.append({'sha256':dg,'offset':off,'length':ln}); chunks.append(data); off+=ln
    header=HEADER.pack(MAGIC,FORMAT_VERSION,HEADER_SIZE,count,RECORD_SIZE,payload_offset)
    index=b''.join(RECORD.pack(bytes.fromhex(r['sha256']),r['offset'],r['length']) for r in records)
    pack=header+index+b''.join(chunks)
    if len(pack)!=off:raise PackError('pack_length_internal')
    meta={'catalog':cat,'index_bytes':index,'payload_bytes':pack[payload_offset:]}
    return pack,records,meta

def build_index(root:Path,pack:bytes,records:list[dict[str,Any]],meta:dict[str,Any])->dict[str,Any]:
    cat=meta['catalog']; payload_offset=HEADER_SIZE+len(records)*RECORD_SIZE
    core={'schema':'memory-patch-object-pack-index/1','pack_format':{'magic':'MPOP','version':FORMAT_VERSION,'byte_order':'big-endian','header_size':HEADER_SIZE,'record_size':RECORD_SIZE,'max_objects':MAX_OBJECTS,'max_object_size_bytes':MAX_OBJECT_SIZE,'max_pack_size_bytes':MAX_PACK_SIZE,'record_layout':'32-byte SHA-256 digest + uint64 absolute offset + uint64 length; records sorted by full lowercase hex digest; payload ranges contiguous with no gaps or trailing bytes'},'catalog_id':cat['catalog_id'],'catalog_digest_sha256':cat['catalog_digest_sha256'],'pack_path':PACK_REL,'object_count':len(records),'payload_offset':payload_offset,'pack_size_bytes':len(pack),'pack_sha256':sha_bytes(pack),'binary_index_sha256':sha_bytes(meta['index_bytes']),'payload_region_sha256':sha_bytes(meta['payload_bytes']),'records':records}
    dg=sha_bytes(canonical_bytes(core))
    return {**core,'index_digest_sha256':dg,'pack_id':'pack-'+dg[:20],'truth_boundary':'Local deterministic pack/index byte-integrity evidence only; the loose SHA-256 object store remains authoritative. Not signing, authenticated backup, timestamp authority, public witness consensus, production disaster recovery, deployment, domain control, or real-world authorization.'}

def write_pack(root:Path)->dict[str,Any]:
    pack,records,meta=build_pack_bytes(root); idx=build_index(root,pack,records,meta)
    pp=root/PACK_REL; pp.parent.mkdir(parents=True,exist_ok=True); pp.write_bytes(pack)
    ip=root/INDEX_REL; ip.parent.mkdir(parents=True,exist_ok=True); ip.write_text(json.dumps(idx,ensure_ascii=False,indent=2)+'\n','utf-8')
    return idx

def parse_pack_bytes(data:bytes,verify_payloads:bool=True)->dict[str,Any]:
    if len(data)<HEADER_SIZE:raise PackError('truncated_header')
    magic,version,hsize,count,rsize,payload_offset=HEADER.unpack_from(data,0)
    if magic!=MAGIC:raise PackError('bad_magic')
    if version!=FORMAT_VERSION:raise PackError('bad_version')
    if hsize!=HEADER_SIZE:raise PackError('bad_header_size')
    if rsize!=RECORD_SIZE:raise PackError('bad_record_size')
    if count>MAX_OBJECTS:raise PackError('object_count_bound')
    expected_payload=HEADER_SIZE+count*RECORD_SIZE
    if payload_offset!=expected_payload:raise PackError('payload_offset')
    if len(data)>MAX_PACK_SIZE:raise PackError('pack_size_bound')
    if len(data)<payload_offset:raise PackError('truncated_index')
    records=[]; last=''; expected_off=payload_offset
    for i in range(count):
        pos=HEADER_SIZE+i*RECORD_SIZE; raw_dg,off,ln=RECORD.unpack_from(data,pos); dg=raw_dg.hex()
        if last and dg<=last:raise PackError('duplicate_or_unsorted_digest')
        last=dg
        if ln>MAX_OBJECT_SIZE:raise PackError('object_size_bound')
        if off!=expected_off:raise PackError('noncontiguous_or_overlapping_range')
        if off>0xffffffffffffffff-ln:raise PackError('uint64_overflow')
        end=off+ln
        if end>len(data):raise PackError('range_out_of_bounds')
        payload=data[off:end]
        if verify_payloads and sha_bytes(payload)!=dg:raise PackError('payload_digest_mismatch:'+dg)
        records.append({'sha256':dg,'offset':off,'length':ln}); expected_off=end
    if expected_off!=len(data):raise PackError('trailing_bytes')
    return {'magic':'MPOP','version':version,'object_count':count,'payload_offset':payload_offset,'records':records,'binary_index_sha256':sha_bytes(data[HEADER_SIZE:payload_offset]),'payload_region_sha256':sha_bytes(data[payload_offset:]),'pack_sha256':sha_bytes(data),'pack_size_bytes':len(data)}

def verify_index(root:Path,index:dict[str,Any]|None=None,pack_bytes:bytes|None=None)->dict[str,Any]:
    idx=index or load(root/INDEX_REL); data=pack_bytes if pack_bytes is not None else (root/idx.get('pack_path',PACK_REL)).read_bytes()
    parsed=parse_pack_bytes(data,True); cat,objs=catalog_objects(root)
    if idx.get('schema')!='memory-patch-object-pack-index/1':raise PackError('index_schema')
    fmt=idx.get('pack_format') or {}
    expected_fmt={'magic':'MPOP','version':FORMAT_VERSION,'byte_order':'big-endian','header_size':HEADER_SIZE,'record_size':RECORD_SIZE,'max_objects':MAX_OBJECTS,'max_object_size_bytes':MAX_OBJECT_SIZE,'max_pack_size_bytes':MAX_PACK_SIZE,'record_layout':'32-byte SHA-256 digest + uint64 absolute offset + uint64 length; records sorted by full lowercase hex digest; payload ranges contiguous with no gaps or trailing bytes'}
    if fmt!=expected_fmt:raise PackError('format_metadata')
    if idx.get('catalog_id')!=cat['catalog_id'] or idx.get('catalog_digest_sha256')!=cat['catalog_digest_sha256']:raise PackError('catalog_binding')
    for k in ('object_count','payload_offset','pack_size_bytes','pack_sha256','binary_index_sha256','payload_region_sha256','records'):
        if idx.get(k)!=parsed[k]:raise PackError('index_pack_substitution:'+k)
    expected=[{'sha256':o['sha256'],'offset':r['offset'],'length':o['size_bytes']} for o,r in zip(objs,parsed['records'])]
    if expected!=parsed['records']:raise PackError('catalog_record_mismatch')
    core={k:idx[k] for k in ('schema','pack_format','catalog_id','catalog_digest_sha256','pack_path','object_count','payload_offset','pack_size_bytes','pack_sha256','binary_index_sha256','payload_region_sha256','records')}
    dg=sha_bytes(canonical_bytes(core))
    if idx.get('index_digest_sha256')!=dg or idx.get('pack_id')!='pack-'+dg[:20]:raise PackError('index_digest')
    return {'pass':True,'pack_id':idx['pack_id'],'index_digest_sha256':dg,'pack_sha256':parsed['pack_sha256'],'binary_index_sha256':parsed['binary_index_sha256'],'payload_region_sha256':parsed['payload_region_sha256'],'object_count':parsed['object_count'],'pack_size_bytes':parsed['pack_size_bytes']}

def payload_for(data:bytes,record:dict[str,Any])->bytes:
    return data[record['offset']:record['offset']+record['length']]

def extract(root:Path,destination:Path,digests:Iterable[str]|None=None,index:dict[str,Any]|None=None)->dict[str,Any]:
    idx=index or load(root/INDEX_REL); data=(root/idx['pack_path']).read_bytes(); verify_index(root,idx,data)
    wanted=None if digests is None else list(digests)
    if wanted is not None:
        if len(wanted)!=len(set(wanted)):raise PackError('duplicate_sparse_digest')
        avail={r['sha256'] for r in idx['records']}
        if any(d not in avail for d in wanted):raise PackError('unknown_sparse_digest')
        wanted=set(wanted)
    if destination.exists() and any(destination.iterdir()):raise PackError('destination_not_fresh')
    destination.mkdir(parents=True,exist_ok=True); n=0; total=0
    for r in idx['records']:
        if wanted is not None and r['sha256'] not in wanted:continue
        payload=payload_for(data,r); out=destination/extracted_rel(r['sha256']); out.parent.mkdir(parents=True,exist_ok=True); out.write_bytes(payload); n+=1;total+=len(payload)
    return {'pass':True,'extracted_object_count':n,'extracted_bytes':total,'sparse':wanted is not None}

def verify_extraction_against_loose(root:Path,destination:Path,digests:Iterable[str]|None=None)->dict[str,Any]:
    idx=load(root/INDEX_REL); wanted={r['sha256'] for r in idx['records']} if digests is None else set(digests); mismatches=[]
    for dg in sorted(wanted):
        a=destination/extracted_rel(dg); b=root/loose_rel(dg)
        if not a.is_file() or not b.is_file() or a.read_bytes()!=b.read_bytes():mismatches.append(dg)
    return {'pass':not mismatches,'checked':len(wanted),'mismatches':mismatches}

def materialize_bundles_from_pack(root:Path)->dict[str,Any]:
    idx=load(root/INDEX_REL); data=(root/idx['pack_path']).read_bytes();verify_index(root,idx,data); by={r['sha256']:r for r in idx['records']}; refs=load(root/REFS_REL); bundles=[]
    for b in refs['bundles']:
        with tempfile.TemporaryDirectory() as td:
            dest=Path(td); count=0
            for ref in b['references']:
                dg=ref['object_sha256']; payload=payload_for(data,by[dg]); rel=ref['path']; out=dest/rel;out.parent.mkdir(parents=True,exist_ok=True);out.write_bytes(payload);count+=1
            # deterministic tree digest uses path\0sha256\n
            entries=[]
            for p in sorted(x for x in dest.rglob('*') if x.is_file()):entries.append(f"{p.relative_to(dest).as_posix()}\0{sha_file(p)}\n")
            tree=sha_bytes(''.join(entries).encode())
            preserved=None
            for br in refs['bundles']:
                if br['bundle_id']==b['bundle_id']:
                    manifest=br['bundle_manifest']; break
            bm=load(root/manifest); payload_root=None
            # map manifest to checked-in payload root via object-store constants
            import sys;sys.path.insert(0,str(root/'tools'));import memory_patch_object_store as osmod
            for m,pr in osmod.BUNDLES:
                if m==manifest:payload_root=pr;break
            if payload_root is None:raise PackError('unknown_bundle_manifest')
            old=[]
            for p in sorted(x for x in (root/payload_root).rglob('*') if x.is_file()):old.append(f"{p.relative_to(root/payload_root).as_posix()}\0{sha_file(p)}\n")
            oldtree=sha_bytes(''.join(old).encode())
            bundles.append({'bundle_id':b['bundle_id'],'file_count':count,'tree_digest_sha256':tree,'preserved_tree_digest_sha256':oldtree,'byte_for_byte_parity':tree==oldtree})
    core={'schema':'memory-patch-pack-materialization/1','pack_id':idx['pack_id'],'bundles':bundles}
    dg=sha_bytes(canonical_bytes(core));return {**core,'materialization_digest_sha256':dg,'pass':all(x['byte_for_byte_parity'] for x in bundles)}

def loose_status(root:Path,store_root:Path|None=None)->dict[str,Any]:
    cat=load(root/CATALOG_REL); base=store_root if store_root is not None else root/LOOSE_REL; missing=[];corrupt=[];healthy=[]
    for o in cat['objects']:
        dg=o['sha256']; p=base/dg[:2]/dg
        if not p.is_file():missing.append(dg)
        elif p.stat().st_size!=o['size_bytes'] or sha_file(p)!=dg:corrupt.append(dg)
        else:healthy.append(dg)
    return {'expected_object_count':len(cat['objects']),'healthy_object_count':len(healthy),'missing_objects':missing,'corrupt_objects':corrupt,'pass':not missing and not corrupt}

def build_scrub(root:Path)->dict[str,Any]:
    import sys;sys.path.insert(0,str(root/'tools'));import memory_patch_object_store as osmod
    cat=load(root/CATALOG_REL);refs=load(root/REFS_REL);ret=load(root/RETENTION_REL);cp=load(root/CHECKPOINT_REL)
    ov=osmod.validate_catalog(root,cat,refs);rv=osmod.validate_retention(cat,ret);cv=osmod.validate_checkpoint(cp);pv=verify_index(root); loose=loose_status(root); mat=materialize_bundles_from_pack(root)
    core={'schema':'memory-patch-pack-scrub-results/1','pack_id':pv['pack_id'],'catalog_id':cat['catalog_id'],'loose_store':loose,'pack':pv,'object_store_catalog':ov,'retention':rv,'object_checkpoint':cv,'bundle_materialization':mat,'automatic_delete_count':0}
    dg=sha_bytes(canonical_bytes(core));return {**core,'scrub_digest_sha256':dg,'scrub_id':'scrub-'+dg[:20],'pass':all([loose['pass'],pv['pass'],mat['pass']]),'truth_boundary':'Local scrub compares checked-in bytes/digests only. It does not authenticate authorship, backup custody, external replicas, timestamps, public consensus, deployment, or authority.'}

def repair_loose_from_pack(root:Path,store_root:Path,selected:Iterable[str]|None=None,simulate_interrupt_digest:str|None=None)->dict[str,Any]:
    idx=load(root/INDEX_REL);data=(root/idx['pack_path']).read_bytes();verify_index(root,idx,data);by={r['sha256']:r for r in idx['records']};cat=load(root/CATALOG_REL);valid={o['sha256'] for o in cat['objects']}
    targets=sorted(valid if selected is None else set(selected))
    if any(d not in valid for d in targets):raise PackError('repair_unknown_digest')
    actions=[]
    for dg in targets:
        p=store_root/dg[:2]/dg;state='healthy'
        if not p.is_file():state='missing'
        elif sha_file(p)!=dg or p.stat().st_size!=by[dg]['length']:state='corrupt'
        if state=='healthy':continue
        payload=payload_for(data,by[dg]);
        if sha_bytes(payload)!=dg:raise PackError('repair_pack_payload_digest')
        p.parent.mkdir(parents=True,exist_ok=True);tmp=p.with_name('.'+dg+'.repair.tmp')
        tmp.write_bytes(payload)
        if sha_file(tmp)!=dg:tmp.unlink(missing_ok=True);raise PackError('repair_staged_verify')
        if simulate_interrupt_digest==dg:
            tmp.unlink(missing_ok=True);raise PackError('simulated_repair_interruption')
        os.replace(tmp,p)
        if sha_file(p)!=dg:raise PackError('repair_final_verify')
        actions.append({'sha256':dg,'prior_state':state,'result':'reconstructed_from_verified_pack'})
    after=loose_status(root,store_root)
    return {'pass':after['pass'],'repaired_object_count':len(actions),'actions':actions,'after':after,'automatic_delete_count':0}

def build_repair_fixture(root:Path)->dict[str,Any]:
    cat=load(root/CATALOG_REL); pick=[cat['objects'][0]['sha256'],cat['objects'][len(cat['objects'])//2]['sha256'],cat['objects'][-1]['sha256']]
    with tempfile.TemporaryDirectory() as td:
        base=Path(td)/'sha256';
        # start from complete extracted pack store
        extract(root,Path(td)/'extract')
        shutil.copytree(Path(td)/'extract'/'sha256',base)
        (base/pick[0][:2]/pick[0]).unlink(); (base/pick[1][:2]/pick[1]).write_bytes(b'corrupt-fixture')
        before=loose_status(root,base); repair=repair_loose_from_pack(root,base,pick[:2]); parity=loose_status(root,base)
    with tempfile.TemporaryDirectory() as td2:
        base2=Path(td2)/'sha256';extract(root,Path(td2)/'extract');shutil.copytree(Path(td2)/'extract'/'sha256',base2);(base2/pick[2][:2]/pick[2]).unlink()
        interrupted=False
        try:repair_loose_from_pack(root,base2,[pick[2]],simulate_interrupt_digest=pick[2])
        except PackError as e:interrupted=str(e)=='simulated_repair_interruption'
        untouched=not (base2/pick[2][:2]/pick[2]).exists()
    core={'schema':'memory-patch-pack-repair-results/1','pack_id':load(root/INDEX_REL)['pack_id'],'fixture_digests':pick,'before_missing_count':len(before['missing_objects']),'before_corrupt_count':len(before['corrupt_objects']),'repair':repair,'final_parity_pass':parity['pass'],'interruption_fixture_pass':interrupted and untouched,'automatic_delete_count':0,'protected_anchor_mutation_count':0}
    dg=sha_bytes(canonical_bytes(core));return {**core,'repair_result_digest_sha256':dg,'pass':repair['pass'] and parity['pass'] and interrupted and untouched,'truth_boundary':'Repair result is an isolated local fixture only. The checked-in loose store is not mutated by this fixture; protected anchors and historical payloads are never repair targets.'}

def build_all(root:Path)->dict[str,Any]:
    idx=write_pack(root);scrub=build_scrub(root);repair=build_repair_fixture(root)
    (root/SCRUB_REL).write_text(json.dumps(scrub,ensure_ascii=False,indent=2)+'\n','utf-8')
    (root/REPAIR_REL).write_text(json.dumps(repair,ensure_ascii=False,indent=2)+'\n','utf-8')
    return {'index':idx,'scrub':scrub,'repair':repair}

def main():
    ap=argparse.ArgumentParser();ap.add_argument('--root',default=str(ROOT));ap.add_argument('--build',action='store_true');ap.add_argument('--verify-only',action='store_true');ap.add_argument('--extract');ap.add_argument('--digest',action='append');args=ap.parse_args();root=Path(args.root).resolve()
    if args.build:out=build_all(root)
    elif args.extract:out=extract(root,Path(args.extract),args.digest)
    else:out={'pack':verify_index(root),'scrub':build_scrub(root)}
    print(json.dumps(out,ensure_ascii=False))
if __name__=='__main__':main()