111 lines
7.2 KiB
Python
111 lines
7.2 KiB
Python
"""Build the versioned CC-BY catalogue from official SRD PDFs using PyMuPDF.
|
||
Usage: python scripts/import-srd.py /path/to/srd-sources
|
||
PDFs are 2014.pdf (SRD 5.1) and 2024.pdf (SRD 5.2.1).
|
||
"""
|
||
import fitz,re,json,gzip,hashlib,sys,unicodedata
|
||
from pathlib import Path
|
||
root=Path(sys.argv[1]);out=Path(__file__).resolve().parents[1]/'server/data'
|
||
def clean(s):
|
||
s=s.replace('\xad','').replace('\u2010','-').replace('\u2011','-').replace('\u2013','–')
|
||
return re.sub(r'-{2,}','-',re.sub(r'\s+',' ',s)).strip()
|
||
def lines(doc,start,end):
|
||
result=[]
|
||
for page in range(start-1,end):
|
||
p=doc[page];rows=[]
|
||
for b in p.get_text('dict')['blocks']:
|
||
for l in b.get('lines',[]):
|
||
ss=l['spans'];t=clean(''.join(s['text'] for s in ss))
|
||
if not t or 'System Reference Document' in t or t.isdigit():continue
|
||
if l['bbox'][1]>p.rect.height-35:continue
|
||
rows.append({'text':t,'size':max(s['size'] for s in ss),'font':ss[0]['font'],'page':page+1,'x':l['bbox'][0],'y':l['bbox'][1]})
|
||
result+=sorted(rows,key=lambda l:(l['x']>=p.rect.width/2,l['y'],l['x']))
|
||
return result
|
||
def blocks(rows,size=12):
|
||
entries=[];name=[];body=[];page=0;in_body=False
|
||
def flush():
|
||
if name:entries.append({'name':' '.join(name),'description':'\n'.join(body),'page':page})
|
||
for l in rows:
|
||
heading=abs(l['size']-size)<.25 and 'GillSans-SemiBold' in l['font']
|
||
if heading:
|
||
if in_body:flush();name=[];body=[];in_body=False
|
||
if not name:page=l['page']
|
||
name.append(l['text'])
|
||
elif name:
|
||
body.append(l['text']);in_body=True
|
||
flush();return entries
|
||
catalogue=[];sources=[]
|
||
for edition,version,spell_range,item_range in [('2014','5.1',(114,193),(207,253)),('2024','5.2.1',(107,175),(209,253))]:
|
||
path=root/(edition+'.pdf');doc=fitz.open(path);url=f'https://media.dndbeyond.com/compendium-images/srd/{"5.1" if edition=="2014" else "5.2"}/SRD_CC_v{version}.pdf'
|
||
sources.append({'edition':edition,'version':version,'url':url,'sha256':hashlib.sha256(path.read_bytes()).hexdigest(),'license':'CC-BY-4.0','licenseUrl':'https://creativecommons.org/licenses/by/4.0/legalcode','attribution':('This work includes '+clean(doc[0].get_text()).split('This work includes')[1].split('Please do not')[0].strip()).replace('/4.0/ legalcode','/4.0/legalcode')})
|
||
def add(kind,e,**extra):
|
||
name=e['name'];slug=re.sub(r'[^a-z0-9]+','-',unicodedata.normalize('NFKD',name).encode('ascii','ignore').decode().lower()).strip('-')
|
||
catalogue.append({'id':f'{edition}:{kind}:{extra.get("category", "").lower().replace(" ","-")+":" if kind=="item" else ""}{slug}','edition':edition,'kind':kind,'name':name,'description':e['description'],'source':f'SRD {version}','sourceUrl':url,'page':e['page'],**extra})
|
||
for e in blocks(lines(doc,*spell_range)):
|
||
meta=e['description'].split('\n')[0]
|
||
match=re.match(r'Level ([0-9]) (\w+)',meta) if edition=='2024' else re.match(r'([1-9])(?:st|nd|rd|th)-+level (\w+)',meta)
|
||
cantrip=('Cantrip' in meta if edition=='2024' else 'cantrip' in meta)
|
||
if not match and not cantrip:continue
|
||
level=int(match[1]) if match else 0
|
||
if 'Casting Time:' not in e['description'] or 'Duration:' not in e['description']:raise ValueError(e['name'])
|
||
add('spell',e,level=level)
|
||
for e in blocks(lines(doc,*item_range)):
|
||
if re.match(r'(?:Wondrous [Ii]tem|Weapon|Armor|Ring|Rod|Staff|Wand|Potion|Scroll|Ammunition)',e['description']):add('item',e,category='Magic item')
|
||
if edition=='2014':
|
||
allowed=['Attack','Cast a Spell','Dash','Disengage','Dodge','Help','Hide','Ready','Search','Use an Object']
|
||
for e in blocks(lines(doc,93,94),13.92):
|
||
if e['name'] in allowed:add('action',e)
|
||
else:
|
||
for e in blocks(lines(doc,176,191)):
|
||
if '[Action]' in e['name']:e['name']=e['name'].replace(' [Action]','');add('action',e)
|
||
# Weapon tables: use explicit printed columns, rather than infer character bonuses.
|
||
page=66 if edition=='2014' else 91
|
||
p=doc[page-1]
|
||
for table in p.find_tables(strategy='text').tables:
|
||
for row in table.extract():
|
||
cells=[clean(x or '') for x in row]
|
||
if edition=='2014' and len(cells)>=5 and re.match(r'\d+d\d+',cells[2]):
|
||
name,cost,damage,weight=cells[:4];properties=''.join(cells[4:]);extra={'damage':damage,'cost':cost,'weight':weight,'properties':properties}
|
||
elif edition=='2024' and len(cells)>=8 and re.match(r'\d+d\d+',cells[1]):
|
||
name,damage=cells[:2];extra={'damage':damage,'properties':' '.join(x for x in cells[2:-3] if x),'mastery':cells[-3],'weight':cells[-2],'cost':cells[-1]}
|
||
else:continue
|
||
add('item',{'name':name,'description':'\n'.join(f'{k.title()}: {v}' for k,v in extra.items()),'page':page},category='Weapon',**extra)
|
||
# Revised gear has named descriptions, including its printed price.
|
||
if edition=='2024':
|
||
for e in blocks(lines(doc,95,101)):
|
||
if re.search(r'\([\d,]+ (?:CP|SP|GP)\)$',e['name']):
|
||
price=re.search(r'\(([^)]+)\)$',e['name'])[1];e['name']=re.sub(r' \([^)]+\)$','',e['name']);add('item',e,category='Adventuring gear',cost=price)
|
||
# Standard armor tables, preserving printed values without calculating AC.
|
||
armor_names={'Padded','Leather','Studded leather','Hide','Chain shirt','Scale mail','Breastplate','Half plate','Ring mail','Chain mail','Splint','Plate','Shield','Padded Armor','Leather Armor','Studded Leather Armor','Hide Armor','Chain Shirt','Scale Mail','Half Plate Armor','Ring Mail','Chain Mail','Splint Armor','Plate Armor'}
|
||
for armor_page in ([63,64] if edition=='2014' else [92]):
|
||
p=doc[armor_page-1];groups={}
|
||
for b in p.get_text('dict')['blocks']:
|
||
for l in b.get('lines',[]):
|
||
t=clean(''.join(x['text'] for x in l['spans']));groups.setdefault(round(l['bbox'][1]),[]).append((l['bbox'][0],t))
|
||
for group in groups.values():
|
||
cells=[t for x,t in sorted(group)];name=cells[0] if cells else '';body=' · '.join(cells[1:])
|
||
if name in armor_names and re.search(r'\d[\d,]* (?:gp|GP|sp|SP)',body):add('item',{'name':name,'description':body,'page':armor_page},category='Armor')
|
||
if edition=='2014':
|
||
p=doc[68]
|
||
for left,right,cost_x,weight_x in [(50,300,170,210),(320,570,440,480)]:
|
||
groups={}
|
||
for b in p.get_text('dict')['blocks']:
|
||
for l in b.get('lines',[]):
|
||
x,y=l['bbox'][:2]
|
||
if left<=x<right and y<755:
|
||
groups.setdefault(round(y),[]).append((x,clean(''.join(s['text'] for s in l['spans']))))
|
||
for row in groups.values():
|
||
name=' '.join(t for x,t in sorted(row) if x<cost_x);cost=' '.join(t for x,t in sorted(row) if cost_x<=x<weight_x);weight=' '.join(t for x,t in sorted(row) if x>=weight_x)
|
||
if name and re.fullmatch(r'[\d,.]+ (?:cp|sp|gp|pp)',cost):add('item',{'name':name,'description':f'Cost: {cost}\nWeight: {weight}','page':69},category='Adventuring gear',cost=cost,weight=weight)
|
||
|
||
keys=[e['id'] for e in catalogue]
|
||
if len(keys)!=len(set(keys)):
|
||
from collections import Counter
|
||
raise ValueError([k for k,v in Counter(keys).items() if v>1])
|
||
for edition in ['2014','2024']:
|
||
for kind in ['spell','action','item']:
|
||
entries=[e for e in catalogue if e['edition']==edition and e['kind']==kind];print(edition,kind,len(entries),[e['name'] for e in entries[:5]])
|
||
payload={'seedVersion':1,'sources':sources,'entries':catalogue}
|
||
out.mkdir(exist_ok=True);(out/'srd-catalogue.json.gz').write_bytes(gzip.compress(json.dumps(payload,ensure_ascii=False,separators=(',',':')).encode(),mtime=0))
|
||
(root/'catalogue-preview.json').write_text(json.dumps(payload,ensure_ascii=False,indent=2))
|
||
print('gzip bytes',(out/'srd-catalogue.json.gz').stat().st_size)
|