"""Collect a bounded number of conventional AAHub text-art records. Only public, current-indexed MLT JSON is read. No images, source scripts, generated permutations, deleted/unreferenced objects or user uploads are used. """ import argparse,collections,concurrent.futures,gzip,hashlib,json,re,subprocess,time,unicodedata,urllib.request,urllib.error,threading from pathlib import Path parser=argparse.ArgumentParser();parser.add_argument('--inventory',type=Path,required=True);parser.add_argument('--cache',type=Path,required=True);parser.add_argument('--output',type=Path,required=True);parser.add_argument('--count',type=int,required=True);parser.add_argument('--workers',type=int,default=6);args=parser.parse_args() if args.count<1 or args.count>1000000:raise SystemExit('A positive bounded target is required') args.cache.mkdir(parents=True,exist_ok=True);args.output.mkdir(parents=True,exist_ok=True);(args.output/'parts').mkdir(exist_ok=True) inv=json.loads(args.inventory.read_text());base_manifest=json.loads(Path('data/kaomoji-manifest.json').read_text()); if base_manifest.get('collection_stopped'):raise SystemExit('Collection is stopped at its target; no further collection is enabled.') base_rows=[r for f in base_manifest['files'] for r in json.loads(Path(f).read_text())] curated=json.loads(subprocess.check_output(['node','--input-type=module','-e',"import {curated} from './catalog.js';console.log(JSON.stringify(curated))"])) def canonical(s):return ''.join(unicodedata.normalize('NFC',s).replace('\ufeff','').replace('\u200b','').replace('\u2060','').split()) def digest(s):return hashlib.sha256(canonical(s).encode()).digest() seen={digest(r[0]) for r in base_rows+curated+json.loads(Path('data/emoji.json').read_text())+json.loads(Path('data/unicode.json').read_text())};del base_rows adult=re.compile(r'R\s*[-_]?\s*1[578]|１８禁|エロ|成人|アダルト|性的|性器|全裸|ヌード|裸体|性交|性行為|セックス|陵辱|凌辱|レイプ|孕ませ|巨乳化|爆乳|陰茎|精液|フェラ|クンニ|アナル|パイズリ|オナニー|自慰|ふたなり',re.I) priority={'汎用AA':0,'2ch':1,'企業・ご当地キャラクター':2} files=[f for f in inv['files'] if not adult.search(unicodedata.normalize('NFKC',f['path']))] files.sort(key=lambda f:(priority.get(f['path'].split('/')[0],3),-f['size'],f['path'])) ua='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';clock_lock=threading.Lock();next_request=0 stats=collections.Counter();families=[];start=time.time();raw_provenance=[] def fetch(f): global next_request target=args.cache/(f['key'].split('/')[-1]+'.gz') if target.exists(): raw=target.read_bytes() if len(raw)==f['size'] and hashlib.md5(raw).hexdigest()==f['etag']:return raw,True with clock_lock: delay=max(0,next_request-time.monotonic());next_request=max(next_request,time.monotonic())+.08 if delay:time.sleep(delay) url=f['url']+'?generation='+f['generation'];req=urllib.request.Request(url,headers={'User-Agent':ua,'Accept-Encoding':'gzip'}) with urllib.request.urlopen(req,timeout=40) as response:raw=response.read() if len(raw)!=f['size'] or hashlib.md5(raw).hexdigest()!=f['etag']:raise ValueError('Source integrity mismatch: '+f['key']) data=gzip.decompress(raw) if raw[:2]==b'\x1f\x8b' else raw parsed=json.loads(data) if not isinstance(parsed.get('aa'),list):raise ValueError('Unexpected source schema') tmp=target.with_suffix('.tmp');tmp.write_bytes(raw);tmp.replace(target);return raw,False def write_family(f,records,source_count,source_sha): if not records:return family_id=f['key'].split('/')[-1].removesuffix('.json');parts=[] for i in range(0,len(records),128): name=family_id+'.'+str(i//128)+'.json';raw=json.dumps(records[i:i+128],ensure_ascii=False,separators=(',',':')).encode() if len(raw)>24*1024*1024:raise ValueError('Generated shard exceeds safe static-asset limit') (args.output/'parts'/name).write_bytes(raw);parts.append({'file':'./data/archive/parts/'+name,'start':i,'count':len(records[i:i+128]),'bytes':len(raw),'sha256':hashlib.sha256(raw).hexdigest()}) aliases={'汎用AA':'通用 图案','2ch':'表情 角色 论坛','企業・ご当地キャラクター':'吉祥物 标志'} families.append({'id':family_id,'name':f['title'],'path':f['path'],'group':f['path'].split('/')[0],'tags':aliases.get(f['path'].split('/')[0],'')+' '+f['path'],'count':len(records),'source_rows':source_count,'source':'https://aahub.org/mlt/'+family_id,'raw_url':f['url'],'generation':f['generation'],'compressed_md5':f['etag'],'source_sha256':source_sha,'parts':parts}) def checkpoint(done=False): status={'accepted':stats['accepted'],'target':args.count,'catalogue_total':base_manifest['total']+stats['accepted'],'families':len(families),'stats':dict(stats),'elapsed_seconds':round(time.time()-start),'complete':done} (args.output/'status.json').write_text(json.dumps(status,ensure_ascii=False,indent=2));print(json.dumps(status,ensure_ascii=False),flush=True) try: with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as executor: pending=collections.deque();cursor=0 for _ in range(args.workers*2): if cursor=args.count:stats['unprocessed_tail']+=len(entries)-i;break stats['scanned']+=1;text=item.get('value','').replace('\r\n','\n').replace('\r','\n').expandtabs(8);text='\n'.join(line.rstrip() for line in text.split('\n')).strip('\n');body=''.join(text.split());normal=unicodedata.normalize('NFKC',text) if not body or any(unicodedata.category(c)=='Cc' and c!='\n' for c in text):stats['invalid']+=1;continue symbols=sum(unicodedata.category(c)[0] in 'SP' for c in body);lines=[l for l in text.split('\n') if l.strip()] is_art=(len(lines)>=2 and symbols>=8 and symbols/len(body)>=.18) or (len(lines)==1 and len(body)<=120 and symbols>=4 and symbols/len(body)>=.45) if not is_art: if len(body)<120 and len(lines)<=2:heading=text.strip()[:100];section_adult=bool(adult.search(unicodedata.normalize('NFKC',heading))) stats['headers_or_prose']+=1;continue if section_adult or adult.search(normal) or item.get('r18'):stats['adult_marked']+=1;continue key=digest(text) if key in seen:stats['duplicates']+=1;continue seen.add(key);family_id=f['key'].split('/')[-1].removesuffix('.json');caption=f['title']+' · '+(heading+' · ' if heading else '')+'#'+str(i+1) records.append([text,caption,'art',f['path'],'aahub_'+family_id+'_'+str(i),'sjis']);stats['accepted']+=1 write_family(f,records,len(entries),hashlib.sha256(raw).hexdigest());stats['processed_files']+=1 if stats['processed_files']%10==0:checkpoint() if cursor