From 6623bd0a18d98accf74e7e5a8897d4a850f37ad6 Mon Sep 17 00:00:00 2001 From: Jesse Griffin Date: Tue, 10 Nov 2026 21:20:50 -0500 Subject: [PATCH 1/2] Add automated data validation for pull requests Add a GitHub Actions workflow that runs on every pull request and on pushes to master. It checks that: - every tracked XML file is well-formed - wlc/ has 39 books and 23,213 verses, and every word has a lemma, a morph code and a unique five-character id - MAPM/MAPM.xml has balanced verse markers and the same verse inventory as wlc/, apart from Josh 21:36-37, which MAM's text does not have - no operating-system junk files are tracked On pull requests it also writes a summary of which words and verses changed to the job summary, to make review quicker. The summary never fails the build. The script uses only the Python standard library. The workflow uses the pull_request trigger with a read-only token, so it is safe for pull requests from forks. Remove the four Thumbs.db files and ignore such files from now on, as proposed in #111, so that the new check passes. Co-Authored-By: Claude Fable 5.2 --- .github/scripts/validate_data.py | 299 ++++++++++++++++++++++++++++ .github/workflows/validate.yml | 31 +++ .gitignore | 5 + HomeFiles/Thumbs.db | Bin 22528 -> 0 bytes OSHB Graphics/Thumbs.db | Bin 10752 -> 0 bytes read/Image/Thumbs.db | Bin 2560 -> 0 bytes structure/OshbVerse/Image/Thumbs.db | Bin 2560 -> 0 bytes 7 files changed, 335 insertions(+) create mode 100644 .github/scripts/validate_data.py create mode 100644 .github/workflows/validate.yml delete mode 100644 HomeFiles/Thumbs.db delete mode 100644 OSHB Graphics/Thumbs.db delete mode 100644 read/Image/Thumbs.db delete mode 100644 structure/OshbVerse/Image/Thumbs.db diff --git a/.github/scripts/validate_data.py b/.github/scripts/validate_data.py new file mode 100644 index 0000000..3b9bfb9 --- /dev/null +++ b/.github/scripts/validate_data.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +"""Validate the OSHB data files and summarise what a change touches. + +Usage: validate_data.py [--root DIR] [--base DIR] + +Checks (any failure exits 1): + * every tracked .xml file is well-formed + * wlc/: 39 books, the expected verse inventory, and every carries a + lemma, a morph code and a unique five-character id + * MAPM/MAPM.xml: verse milestones are balanced and its verse inventory + matches wlc/ (apart from the verses MAM's text does not have) + * no operating-system junk files are tracked + +With --base (a checkout of the revision being merged into) it also writes a +summary of which words and verses changed, to stdout and, on GitHub Actions, +to the job summary. The summary never fails the build; it is there to make +review easier. + +Standard library only. +""" +import argparse +import collections +import os +import re +import subprocess +import sys +import unicodedata +import xml.etree.ElementTree as ET + +NS = "{http://www.bibletechnologies.net/2003/OSIS/namespace}" +WLC_BOOKS = 39 +WLC_VERSES = 23213 +# Verses that BHS numbers but MAM's text does not have. +MAPM_ABSENT = {"Josh.21.36", "Josh.21.37"} +JUNK = re.compile(r"(^|/)(Thumbs\.db|\.DS_Store|desktop\.ini)$", re.I) +MORPH = re.compile(r"^[HA][A-Za-z0-9]+(/[A-Za-z0-9]+)*$") +STRUCT = re.compile(r"^\d+(\.\d+)*$") +WORD_ID = re.compile(r"^[0-9]{2}[A-Za-z0-9]{3}$") +VERSE_ID = re.compile(r"^[1-2]?[A-Za-z]+\.\d+\.\d+$") +MAX_ROWS = 200 + +errors = [] + + +def error(msg): + errors.append(msg) + + +def tracked_files(root): + try: + out = subprocess.run(["git", "-C", root, "ls-files", "-z"], check=True, + capture_output=True).stdout.decode("utf-8") + return [f for f in out.split("\0") if f] + except (OSError, subprocess.CalledProcessError): + found = [] + for base, dirs, files in os.walk(root): + dirs[:] = [d for d in dirs if d != ".git"] + found += [os.path.relpath(os.path.join(base, f), root) for f in files] + return found + + +def parse(path): + try: + return ET.parse(path).getroot() + except ET.ParseError as exc: + error(f"{path}: not well-formed XML: {exc}") + except OSError as exc: + error(f"{path}: cannot read: {exc}") + return None + + +def tag(el): + return el.tag.replace(NS, "") + + +# ---------------------------------------------------------------- wlc/ + +def load_wlc(root, check): + """Return ({word id: record}, [verse ids]) for the wlc/ books under root.""" + words, verses = {}, [] + wlc = os.path.join(root, "wlc") + books = sorted(f for f in os.listdir(wlc) + if f.endswith(".xml") and f != "VerseMap.xml") if os.path.isdir(wlc) else [] + if check and len(books) != WLC_BOOKS: + error(f"wlc/: expected {WLC_BOOKS} book files, found {len(books)}") + for name in books: + rel = f"wlc/{name}" + doc = parse(os.path.join(wlc, name)) + if doc is None: + continue + book = name[:-4] + for verse in doc.iter(NS + "verse"): + vid = verse.get("osisID", "") + verses.append(vid) + if check and (not VERSE_ID.match(vid) or vid.split(".")[0] != book): + error(f"{rel}: unexpected verse osisID {vid!r}") + for w in verse.iter(NS + "w"): + wid = w.get("id") + rec = {"verse": vid, "text": "".join(w.itertext()), + "lemma": w.get("lemma"), "morph": w.get("morph"), + "n": w.get("n"), "type": w.get("type")} + if check: + where = f"{rel} {vid} word {wid!r}" + if wid is None or not WORD_ID.match(wid): + error(f"{where}: missing or malformed id") + elif wid in words: + error(f"{where}: duplicate id (also in {words[wid]['verse']})") + if not rec["lemma"]: + error(f"{where}: missing lemma") + if not rec["morph"] or not MORPH.match(rec["morph"]): + error(f"{where}: missing or malformed morph {rec['morph']!r}") + if rec["n"] is not None and not STRUCT.match(rec["n"]): + error(f"{where}: malformed n {rec['n']!r}") + if not rec["text"].strip(): + error(f"{where}: empty word") + if wid is not None: + words.setdefault(wid, rec) + if check: + if len(verses) != WLC_VERSES: + error(f"wlc/: expected {WLC_VERSES} verses, found {len(verses)}") + if len(set(verses)) != len(verses): + dup = [v for v, c in collections.Counter(verses).items() if c > 1] + error(f"wlc/: duplicate verse ids: {dup[:5]}") + return words, verses + + +# ---------------------------------------------------------------- MAPM/ + +def load_mapm(root, check): + """Return {verse id: text} for MAPM/MAPM.xml, or None if absent/unparseable.""" + path = os.path.join(root, "MAPM", "MAPM.xml") + if not os.path.exists(path): + return None + doc = parse(path) if check else _quiet_parse(path) + if doc is None: + return None + verses, current, skip = collections.OrderedDict(), None, 0 + + def walk(el): + nonlocal current, skip + name = tag(el) + if name == "verse": + if el.get("sID"): + if check and current is not None: + error(f"MAPM/MAPM.xml: verse {el.get('sID')} starts before {current} ends") + current = el.get("sID") + if check and current in verses: + error(f"MAPM/MAPM.xml: duplicate verse {current}") + verses[current] = [] + elif el.get("eID"): + if check and el.get("eID") != current: + error(f"MAPM/MAPM.xml: verse end {el.get('eID')} does not match open verse {current}") + current = None + muted = name in ("note", "title", "header") + skip += muted + if el.text and not skip: + _text(el.text) + for child in el: + walk(child) + if child.tail and not skip: + _text(child.tail) + skip -= muted + + def _text(s): + if current is not None: + verses[current].append(s) + elif check and s.strip(): + error(f"MAPM/MAPM.xml: text outside any verse: {s.strip()[:30]!r}") + + walk(doc) + if check and current is not None: + error(f"MAPM/MAPM.xml: verse {current} is never closed") + return {v: re.sub(r"\s+", " ", "".join(parts)).strip() for v, parts in verses.items()} + + +def _quiet_parse(path): + try: + return ET.parse(path).getroot() + except (ET.ParseError, OSError): + return None + + +def letters(s): + return "".join(c for c in s if "א" <= c <= "ת") + + +# ---------------------------------------------------------------- report + +def report(root, base, words, mapm): + out = ["## Data change summary", ""] + base_words, _ = load_wlc(base, check=False) + added = [i for i in words if i not in base_words] + removed = [i for i in base_words if i not in words] + changed = [] + for wid, rec in words.items(): + old = base_words.get(wid) + if old is None: + continue + fields = [f for f in ("text", "lemma", "morph", "n", "type", "verse") if rec[f] != old[f]] + if fields: + changed.append((wid, rec, old, fields)) + out.append("### wlc/") + if not (added or removed or changed): + out.append("No word-level changes.") + else: + per_field = collections.Counter(f for _, _, _, fs in changed for f in fs) + out.append(f"- words changed: **{len(changed)}** " + f"({', '.join(f'{f}: {n}' for f, n in per_field.most_common()) or 'none'})") + out.append(f"- word ids added: **{len(added)}**, removed: **{len(removed)}**") + if changed: + out += ["", "| verse | id | field | before | after |", "|---|---|---|---|---|"] + rows = [(rec["verse"], wid, f, old[f], rec[f]) + for wid, rec, old, fs in changed for f in fs] + for v, wid, f, a, b in rows[:MAX_ROWS]: + out.append(f"| {v} | `{wid}` | {f} | {a if a is not None else '(none)'} " + f"| {b if b is not None else '(none)'} |") + if len(rows) > MAX_ROWS: + out.append(f"| … | | | {len(rows) - MAX_ROWS} more rows not shown | |") + for label, ids, src in (("added", added, words), ("removed", removed, base_words)): + if ids: + shown = ", ".join(f"`{i}` ({src[i]['verse']})" for i in ids[:30]) + out.append(f"- {label}: {shown}{' …' if len(ids) > 30 else ''}") + out += ["", "### MAPM/"] + base_mapm = load_mapm(base, check=False) + if mapm is None or base_mapm is None: + out.append("MAPM/MAPM.xml is missing or unreadable on one side; not compared.") + elif mapm == base_mapm: + out.append("No verse text changes.") + else: + shared = [v for v in mapm if v in base_mapm] + differ = [v for v in shared if mapm[v] != base_mapm[v]] + marks_only = [v for v in differ + if unicodedata.normalize("NFD", mapm[v]).replace(" ", "") + == unicodedata.normalize("NFD", base_mapm[v]).replace(" ", "")] + cons = [v for v in differ if letters(mapm[v]) != letters(base_mapm[v])] + out.append(f"- verses only in this revision: **{len(set(mapm) - set(base_mapm))}**, " + f"only in base: **{len(set(base_mapm) - set(mapm))}**") + out.append(f"- shared verses whose text differs: **{len(differ)}** of {len(shared)} " + f"({len(marks_only)} differ only in spacing or mark order)") + out.append(f"- of those, verses whose consonants differ: **{len(cons)}**" + + (f" — {', '.join(cons[:60])}{' …' if len(cons) > 60 else ''}" if cons else "")) + return "\n".join(out) + "\n" + + +# ---------------------------------------------------------------- main + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--root", default=".") + ap.add_argument("--base", help="checkout of the base revision, for the change summary") + args = ap.parse_args() + root = args.root + + files = tracked_files(root) + for f in files: + if JUNK.search(f): + error(f"{f}: operating-system junk file is tracked; please remove it") + for f in files: + if f.endswith(".xml") and not f.startswith("wlc/") and f != "MAPM/MAPM.xml": + parse(os.path.join(root, f)) + + words, verses = load_wlc(root, check=True) + mapm = load_mapm(root, check=True) + if mapm is not None and verses: + expected = set(verses) - MAPM_ABSENT + got = set(mapm) + if got != expected: + extra, missing = sorted(got - expected), sorted(expected - got) + error(f"MAPM/MAPM.xml: verse inventory differs from wlc/ " + f"(unexpected: {extra[:8]}{' …' if len(extra) > 8 else ''}; " + f"missing: {missing[:8]}{' …' if len(missing) > 8 else ''})") + + print(f"wlc/: {len(verses)} verses, {len(words)} words") + if mapm is not None: + print(f"MAPM/: {len(mapm)} verses") + + if args.base: + summary = report(root, args.base, words, mapm) + print("\n" + summary) + target = os.environ.get("GITHUB_STEP_SUMMARY") + if target: + with open(target, "a", encoding="utf-8") as fh: + fh.write(summary) + + if errors: + print(f"\nFAILED: {len(errors)} problem(s)") + for e in errors[:100]: + print(f" - {e}") + if os.environ.get("GITHUB_ACTIONS"): + print(f"::error::{e}") + if len(errors) > 100: + print(f" … and {len(errors) - 100} more") + return 1 + print("\nOK: all checks passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 0000000..ae56234 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,31 @@ +name: Validate data + +on: + pull_request: + push: + branches: [master] + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check out the base revision for comparison + if: github.event_name == 'pull_request' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: git worktree add --detach ../base "$BASE_SHA" + + - name: Validate (pull request) + if: github.event_name == 'pull_request' + run: python3 .github/scripts/validate_data.py --base ../base + + - name: Validate (push) + if: github.event_name != 'pull_request' + run: python3 .github/scripts/validate_data.py diff --git a/.gitignore b/.gitignore index 32ba899..2d6383d 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,8 @@ json/ remapped.json hebrew.json + +# Operating-system junk +Thumbs.db +.DS_Store +desktop.ini diff --git a/HomeFiles/Thumbs.db b/HomeFiles/Thumbs.db deleted file mode 100644 index 819482844317d3e6c1240b9f428f2cd5c63adcc0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22528 zcmeFYbx<7Nx9B@ake~?`WEcn*f(8igHV_C9$QRe(!QEXG+}$k+?(XhxGq~H}KFFP% zdwx~t-n!?$Td(dP?^T^O^_jhUGu>;kdiUzxd#|1tf3=oVNwyC7pF|jd0(g4H1fczg zb|ggoFaIX^001(g@^|~``T6<3Hv#~N>VL`qk38^*u=Vfx`wKndB)&lK_nf06z(9bB z;3Wbq1lS005a1%fLxAW4fL91!BOpLPh=2$IF#-|1zzyN;|8pTm z^q7B{g!2Ee+o{vTB}UFT(%;|t{_Ri*;Q)LDa0B!JhJbGXJ^&ZM0P*1j7y);G>Y`k$|VCPW)E0u}^+ZT&C#zvF?wG5T+R zNJQ}Wh>R(+Aus6cIDRN8fN@@u{p}?{9D(Zv5iPR{1K18gCG)A7Sm zI`aR5;yZ~IhNOKa|1S8QGm@C-yNqyYnqD+^b^t)xicHsQUg@t^fUJ~)WSO|W|KIxm zDF3qm`@i?+yq&I&i;U5C{|oyO#{{tdoBe9YcOi(4|DF9W3IEz}CNY|VWBYZMsQ=`+ z?dF;(3@siH6G!rEbYPEaSf?hTSW#&~fp4kit0I;c5dMitQj~YR=0waW&L!`L)M*24 zDni$&KtJEnidx`z_b11UOVVOu4$?O--mF+Z@3u8V>zi$=F#|VjcD9z9PugzA8c$Xn z##<{Zp9#iy@Wuk}35?!tk%i*&p=s!O?(KcwliT3bPOY3;+2bT;}s{{_AT6{=MJ-@BAkY{p&whU08-& z%jenW=T#Sboob^IK_*F#gm;(;jDzX;u^eA_8pXsX_22yn9m&O(`V^1<)5PJOM9Qwf z)0{=d1?0wSZh2SNt1YVkodJk3s)+n^~B==pIkML7L&;qmT# z=a_(q=!-BN9krsQ@ zj1HI6(T+`wDBaeF-<@`LlPE8V8a`HvKBYPINSbmRPns?KoUG0@5Z3<%)lcUJz|6Fo z6mrk9wqs%z7*O`^_j9a}@-;%ZW57Z{25T~>Jp#$3Y3}pbxH!70+bDcwN1=_n-%hXD zd(oVnfVSI1=6p^E6A6a>J~F}{$3uTYG~0zHj5gO=EKvnSb1TD0sd1HKIKHu8twU`<5Fx*c!d9K`cjCRUC=Fc-ou@0~k5ui{{tQ3@0cH#QOs}A1Wp)R%B z@4P9-Tk^aYC1*AzbG?pzZ57%HmH{46=abKd*TX}h3rv~Ytv`Ng4?L@0ajyd zkcqpT7n;#S6BKZA@M^EgxwBIOyfrSOD3c*zQ*T~c1e27Kax-pfvkus^7X@x{mK|z% zeY{2G*UZ|62-ov0ru)^Fg$bL*#t|4F*6nlHbRGwDn%dlt8tGwvYGLf(@90jGi1;FZ z_2FZ2(u%vqwrX#c$tcN2C{}v5sN&%P*Cs#S-aY>)@1&C>MFsA~BHGqe!La@Aa%q8G zHEY#IoKxKr-ZJr|v-Sf03nj%6&Fm1QsjmC*<~_mE69!)^u%x7HX{zFm$OtoG9tPvMtLW*3PB)Jz$yc5n92I~{<@rN&( zzI;J~PEJm?aWB`tTRjri~ zi|bi4<$i5-;70W32S-@*I-T4MBPBuHEL8oaes>Sc{Oz`JpvCxd$0cOmIK02tII%P> zaj^*xHa?-ZHV*Bjojf#3L$zQil^y$jU96XVZVqh??sy7fN2h> zVW^lQ*I@+Th49PDoU1v{x=v@VVd^D~Dx+T9EMhBkr+TmY;lf&+mv{GRuod3}zej@` zHj^Bu!2@N3e~-BM}!bvPJ=J{vb@dB}8wA}1EYteUZzZa5TC;ASj?Df*3Ea@ zXz}Ox)@0fE3l%?mAt+V)ZJ)gPOigvVu)|m#);gfZqbBRp>-dF=Gq)KvONB&^4hu{` zgvkp%JciSB9LhBb(~iPlSmTa==WV)$MUz&~vH0dz9nnLMCZiHLeA=2;A<#c{ZA>bK zM|B^>cl86aU-5s<)fVD&;;)!g{{RMiaDKw5h!If3ZLD_w=w{%m$EaIUG1_jw{fCg% zBMC>WKx=47VwCLDyh3TdQs;wxpQH2O&nwsl4k4?y5`Bx{Kw2MM5Z*G9T0PTeQ*0l0 z>D5$SV||tVxx2rL=p;epncrag{0q~o6>6_(kr?2AT{$GDD&gj0PSiGrSvXx*h zr(w+UIg8-3xeA;u_fW&p;JRTM{BA;MInCC797pCvS!GN`PtTuU4!Ea_s`LQEpq6ms z;rLSsy2lML_D>)BhAE1hRM--&P;-48YoU$OSmk70^jtPG_4-fG=IsEQh_u$pKBEk7 z`@R7;6=$MSNw9R`*B=qRCDHtAWz;xB-m4B1w638i@K!(cWC+L_DyNYCz>rL{<5;Erbi1i2kk@j0o<Ko)VZmv6bLuDNX+O|2Z)f9u<3GNU8iO| zzPNLQsm<|(W)cA;I$Y)D(~W-04rYC{pif6}+k*b%W7O8NC{N4BserPOcd47Of*UQQ zMcn-Jsh)xxSn1te&#hY{L+$akA=|;{&MIrBR7JN zL@cQS;5azb4Wsg{vyo;9RrsM&91Ym{w~vd}pmE`am&rj_d=%r&+^6kPW9b*azoAlZ z?VK0DUimtZ{nDxa^R--6w#s<)ZB(5Fb))@GbevCCZ2NO(38B7dyR~x3-sh>S$q+i5 zHbIkvd)>0${^2wN@Tc{(m+Dy%!Sn2R_9FcB>G4bfHmMPFgMkhPE3ezxKgNbXeul?p zjZOwLKGprHA0SwUEZWr+@ zbuboIEvnus|58G#jwrBfihUKh7c!LaUM6L#cl5})_cjE2{Pye&^V}#3S2rt>NlX0w zini)`idBiy*upP$)*f8vC+3}@plWruyy?qM4=W1c?@L^Kb{qA#Z0+}Uu$Vm458IoO zSqY(4#^=PYruHELA1xetz&?981i|+d`4>4+NK9T1Ui)XRT5eYf)Z9_xwjuZ#de45~ zZ2^ZZ`4q09^m)*kqnX1Tx?H|m`{ecBgT*rsH=Z$L%&VzJxaeZ#1;b*i*FT|NfpWr{ zG#Sk}@kQ>=@pgu2Zg<$bBRVEga|QwKgB6{M8fk*!nMXp(!rHf$vo99rT}kh~t*!DDBFt-PuVxIiyYI90o=HYWswR%J+mAKLH?p*%%!4OLFNO$7 z1=g*aFWR`Re~r2v1{uzm(UG9xrd?d_u!LME(?+s;97oB2`gA7Eo&KTSlNa^QwzK~_ zeX-6dPG1%7bW8 zaqAM9Rd#g+(15bweMG&E+Juk9s$a1pBZ7Rkx=B4A`M#V^DBBttqmTUcEA#LLO?K(Y zU2Ky2m>@Y-S!;((WAyI=_zQ2-#Wps+wc0ZiF0Oz=o~yP;{in%9Js1=-V^xaAk`L#| z#kmvPm>>5ze{1CZO!>}~&)6T?IK#}vl(Jvr&-d6MydGZ(FIqji56Es&%ZlOHhT)aq zjA%+o%SX$nd^0+R$BiV&DU1vIeg+~ZB5fI3#ql|m7!AfQ7CHeTxwb{PF>4Had=5rf zzfTT|P+r2aRa){W&GJ~Sg~|B6H*6Mb;c5;n8oc=i5IdL-_pY!n5exr9g3b;`Uq<5& z9``9Xn*9}-!k5=o^i&oit420lR~}7rGC&OV#vM72f-^ICi1>q8s2I$;@ceBM*5DWm z2s#~Xa(aq+-j!T`^xk854rH`*X{S7`&GCI9azz1W=u0rHdq`+7kQHJ&5Nm9)^`j_w za8e}7!;K_BP3QstC3`;Zw)+{HNcx3n5;bmTA0oV}rG-dgU)a)ann_@Y2~RD#+Maz| zvnI~&Uht^5TBTzdb@JwRpc7L{WFbFR_~~iGy1NwN;WUNa*eGspzEUHWN&~P`^xPEk z=&7}V&J-eD+7_H2mq?u#H~!hXS7&mN>`VDLtVAl*Qa($)sq4|dV&Lg<9yXP`@0E#T ziHB-#&0q@+5ygXs-);}92?1XoBKLS)0U1G_4NKa7BNyjmS$NEE?A8EVV!RH>)*=^9 zDIjXFw{*v^VDNd{qfauBYQ+J=OotbB?^bpf!-OIH%|ne&{D)EZWDdJU#GM0utK5*G z^?{AsGSQ}OnVKf}vd+9tvGXM`$pi-F;+!_V#?Qj{aX5!_>sCpAA%AP5^vqy16}YZT zZgf#EU>n=aHmuosJL2 zIl{V?H5I;e06)0X7`NkddO1(X%BeoEgZEm}SK#|EMd*SdbwdfGF`EA!p}bRp+>5e( z5#77SQHe)t8r=lVovOM*a7i9pHg;Zi7^C}|Ee_&bQwY@ddH$aab-~UN18Q_>O!~?zc?`0}o5#Zc` z_NKNS`DNoQsq9_4`tW&fv6){f88ZIEVQXY@>i5czh5247$zQG&&!p|UP3db(nj?F3 zK$G4({;cG$?DWnNxmVO?Dk1p(uyKl(oIrsr(yJu7KLtu%xWW-l!TW^gcC4j37u;#O zO%E#2ecJ&d$oe&fpd%N4qk?{(NkoS@*7WUxBG148iEfgvqGJtD+=51f6aVq7%TepO z(_wpE(M#<_JMukr)N1UqMx?p7c=cXG!gmr{Z%{phPuGv;TY(hJ&#oDc`@#d&nCcPuKrUL91$=FbWfw9j(e=wx~eykEh>8~kni~5X>R0gj zT)b9*BX!1)k&CaDw=2lx(d!)Ti^6^J{C8SKJ{X~OL`I6)S`~uxn<^mHC;IHf3w8hO z{4^SA_|0+9%cI_Abj<4P?&-UXxmakO*0$G%r%P8zoon<%kf-(L zIuHJ5^@&&b2=4=GbZ`J+qFVa}b+huEenI$)xcPUoFXfE`f}@gn3dia+OrIg^UfdlH z#btTW_%#APW9#218AY5MZ5}7_K@O|f`>3rd$P3=q6K_)P74-kXsc2Zb9tFqVNPD7E z0`9{SOR9D~@d*fMJbZL*oCHAuftK}t#<%z-u8U3FPexJK90o`DLOfCh8lD86Msbly)(5%X&% z_-3}_KK-`L=x?b)Q1E~U zG|y*arB*Fc)o@y$Hu6^cVFTS%A<=8hCN!CTZcey}%%#13Yc5hqQom#+ZB( zYjV0@f)-pWM)3l`V`PgNq(OlC+hhAuo?y5 z-8oJLga;<=@=?gB%t~C}!G$@HYgeF55znFC=HuCAd(BW~bvaimSuJEoUXeVlr9Ow@ ztPU+ZS)`=UM$P6t|B}e+hJ7WBm@{6C%)ny9e#0R!S_9&^{c2+T6XcCUpaVW^q|WK~ z%_&2_)E(v6`&Myz8R@y~$Gu7c(_Qk#w?c-@ObD;9H7lVTTPo%~PFzP9;>Szq*)TXY zctK2GiUJ|$&@Wv$Z9K5)=wRt70#WcDFOPqh{lV+`U{F&TT79@(MpQAjHT&`7W_57* z(c38RyLC0a!?y~P&Tpz%J6h33h)22;FfuuinoR$eqN;~jeZX0dl$|uzuw&!aJ)J*hNUi6V#nuEGa2+#a35vH!6M>l zSrI6$lisnE-nj7IA6kWU6b(~=2JKwMhLEUP#|B{J9tX>sd;P15$gr_~qCud^8q#@8 z+ePGeZWy{_MMtr#oQ91m);#W{aI<%wb%WIz+A54)b*k%v=7p$YzeZFCtEljXWq;81 zVq)>hT&D}*H2+xTQF>8|(CZttB;fuMPbsQ`rS#9LURE ziH0iMKOT}P7p=kEEbH@X>^Jb+W_bB1K92^`SiZO%PQ%jYFyeivpy+DEizjT)`Y`ya@xpP%bZR#11OdG=(uPPfDIx?!i zc-$bHnFmv53%@AV-9f4M+MzLm6DImm@Kp^4+^iDhD&mJs;ML>shzO}42k&=YGfLL) zss%^of;=t_JXTC8ubi~=F1q#`@<~1G;}9X?QuG$*XI;_sEqehpP9ondg`b*7fQpO< zhO3+EcdA?I`Aegnw6j`YBtbP+S< z`cbBwLo`p5tE5=qZs(=(X{#-t&r|Lrnu7bpU9X&)huzDg^%%>(+J>gf2!UV3*@yZe zJv=UV5!$}CPhAO9<|SrG`U#Ov_E?*>3Tg1Y2kJBt*^Pf#_&R3vi?agB;O==bw+zGqaBm$P2=+}jB!&*uI6Y)P{mN-IzWp`7l{2R}#7?kL$I!+YXnOUf z%*n5=|GHDrhkZ5(in?#)Y7V@!!6LXkEKJKnsFUh(!*xhB(7ACX_`%q=D zsp>#Fr``r>_-rCiZV20)D)SgO;P^J6!eQQ}q|xZvB=2FGc?1F8(ogzJF~OSsFdnC= zH+$Id*t0ew2ivK)((!7Lwi>}i`rW>6WIDJLb`V*S2WNJ~!?pL_Wt_(KWa;>vb^KeY zmqQ2JCyV2AwR&XtfShG_r|$zK7Q)e2#k&Jd3Dw5S3@2I|7OJ!F0)S(*PT$q4LCe;6 z)=83u$@kJfNm!Xx7$Ooq-7W^8CLeX*`FQ`}cs3c;rMCi^za1Bx?LGLXSo@2xhM2sYVEqI~7=JjaFuK4N|_a2wm{>c%6CmqkB9Lu&O#lwnZiS86LfsK1m?Zq0D{E=)Ma88HjVRNiJ-WZ=of zZL@x$Tp&#Vw?~C^sWqbGg4hb)&m{>;{TY_9@+fdiC8iMYR<5&zi&?ATWqP)JZ=Gp^ zzZ%rbt&1<7QgRo$@qb((@?L92nKosOmNs$sv4Bt%M_mU;cz>=V6Sys-&ZO{G@u({olLo?MPi{kMkcpA3@ z{O5|ysUqNN|2(CtFP;Ci^#&bud<9j3Ysq}@rt4gD_tKta5 zJ>#~gbx9)a!27jNrr95Ql z$x!&xu5(DgV`gJo{#d*+xis?D^%2r2+Wd^XXVIv;hOQ5s$E}}UwwB>->74&*{laR1 zNVH$uu`vtxXHFil@8UF>^%XjA6|}}(>H4W=(=8CHH zqB0Y3s5TlS)I6`T*bH}AyxDZW^S*d#Ztruj){LxgvB4yvD|sw~RQ~;a`?MO|w#cuK*F(Bw z(?Xf^o0Ayo(t0fqYkKkfxDxk$>~HHQo^#ECgUyxI-N0v2Mo8pgNGPonKi{-F$~JXnjleq4T7EOp zJmfKuzh?)1d4J(fBjxVtn}Cz}ee&T>t`wH+FmXi7JxZ2Wm$xdO4)cO-Mw`p3m|N4T z*`1lxTB#~wa)AxEl_(k5Wb*Bc0u^}3QCFEtJyKP_c48-S*>aljRJT@%xraM_$6K4V z3cD5XWP{~W^KKAQUA#I;3cxap`Gh8RI=Rra+ltU9X33O17B#+NU3u&$V-Tr?Dl-zjtWjFH!| zIs=xlyYeRJJ`u)X8lpozJxd^Uh9J1=1Y0}geU#IojesUR+;*xV$NBLzX7NXbS@Um4 zpqu&9S2vgQH*i+L`%t8gu?{~l?>8 z1r0aLTlrLUJ&mj8or@E1v;?lmo!HxW>dO;)No4myR|{>#YT&$!8p@!%_0zUWmF78u zYjT06w?L{gF&OPM1nsI~oot7uCy8K8ui@Q4Hm!VH9}Sy?J8FJdxGuF8J8RLM499l} zY!Eg@$Pmod;Zg`Pdt3vnZqWazVU@6HKb+f|H$1|#j`C(tNmlm?bV^U=dU?E0dvRJi z{-{R$agO<8@NAa#C=PLwKo$~N&JZBcdBe_?QEp*ynp1)U_&R4^kEEuq01RA516AN+ zki4!#@_-ZyhvcPa3!?^%bLFs*WQ^Zir#ip6&#^K*^&6XZT7E+EwyJ2W%kwKM#>3>f z)h%v%olTuF&K1e^b1tAE+b(0gp?LTnV&bmUuNXy+L+$O(&!)}Sv#gq^!?vVRtEXwE zh9$;#2w%*J3}eNs-@sbjcFgM;9e(%uf#U7@0e-wn|2Xb&ZDugJLQbTf5RXFg$?+O9 zoTuTbA=e6y1jKDx#a(0QDGab9`RGCqE$6UUcUdF_hcXi84mJ2ewX6md)K^GXju|skN*sdcnBiuQsaN>lsja=;H_)LQeGTRjS)<_x zjrYfs3X^7O?!M}M{O;^JUw^@}jnd>fe%av3r_60RLSLi+4!KqQh2CWan%11h(@E;` z%=Q`{NbGYtMT(#@qVdkwK0)np8hha$?}KG2-Ua&_zIkx6@BM<#W~D*mv9g z^yVlcZnvK>x&RAz08h&&VO)5Tp|bM#;Y(zJQ=A(rGm&(B$VBd0v7BuAQmsUp#c3@y z$(tQOg9qK2h386_pmL$TRAcQNOVzi34o*f&)Klc%tCs5zy?>LjWPhqua_L=|9bc@i zn#eNFXfbaZmuU4}X-S)#xxAs&U4&|jx%t>9pY=+0DAt4+&nQ!0zu*u5C zoDUVd(WyQCc?vo=q zVOPW0bi=g{CO9}COV=0~c!LpWOM~ySzi?eM^UGQGxE0<~WA^&C1lYuqni=PDpc|iB zua}5xH74_8V}2{>i=SRkmp(43t#aF~V|zLu0o^A3R`s1FUI!YQ-FE8HRHaUbbuE;2 za8pNNaPDNo=u*zF$Xm?O@n_{t)Kav7tTe9Fn+EdVL7-c}!V|1s9r65rdTqO54XRH= zu8GVNme2v`IcYaMYC3xJBIaG+8;CNNy&{{dS4}}jpR|*7q0cC4z2yGsJR!_J7jPX{ zAbmA_zFXN5B`S_b?WNN~TePHk_b2{+%6+xfS{Z+m@p%pTR`KUW2~XsT9IwSY2@}#Q zab34RdPo{o*le<)97XBO_8lXV%|79Ac%vh=##(DGfI92px8peP-vxw_u-Bp3yrhQX z^$aawzSv$AlM2`6PV09LJ;s6s5bxaO^oJs6kHao#x@q+CcwRQ_+m3xN5|>Os(n;F0 zk4>jw)Uz3Te>2;jEDajbi5{lBf>3s8G3g}if*)9Pnn>cOO%loul}R|9Jt6*(GuFCk4B&)|`sXm4CDn`MgS=3MxB zoWZbotyFIv#V*?h4sS!i~zj%rC-$d_M1s3V%+J3!$VIqqub4QCJ6@RB`$s8EER z$kIK+GWv0B=J$_i#4&9sR6)iUst%IB4`y^K-X*nMy9>KzyU8}*WQI*u3VAa!#|wFI z4l9b7CW1B5yi;c`7|km5PO9yPN9RXAwfa_)UlKeAJXtR!|1(!IN*V|`fyGGOZ0dsSA7hmS0 zNWef&ceN4o-SEM%cq|e_!S;p1V*;UU{^Q&$(xM603A-e&fT*GMY&(%lw8k!=zk4de zrHQ8QS<$8umnR;|nTPQDuVrtN%a*UBeMYraJ9`Fl803@Zk{T}@5Xma6GGpnoIaV9( zm|sliE;mhv#+Yr|$s5o`C+3du?1xYyPJ0x-7ZEd3+%vsJh0c}?N^#bj#vZsN#zzT{ znL}_Ky~olAIbfxE$^#{M&MDJ9U=X_ZKUkD69+cV`7b304mob;_HIvLTY#)lvvnoE+ zf8qWWgc60n5i8_TrSpM=DV0hpEO|oa+Vb}yhVMN9e6;rDGkiucd7hI3(TR|o7TpOjoYgCUtOti?JW zt19t3HxBNhLmte67Gp-3YbQ%kJ`v*yBE`?ewntfV3W0pbiWCvy-dryaI1&5GHZVY; zzXl(OK3BsrZXcNZ4U8vZ&N@2c2A5y^hnZRLna=zDKlx9~Bwmj*ZZ~Op&2mK!=VnR% zud|}vFZKHu>V!_ozU_3jU4JQ3%=j5A=oY&*tEMopgA=kJm>=I2dA4wb)Mmu` zXDN--kg%coEJNM(RSrcJu0{92%|*5cLq3f#|w z^v4fB4XJTvUpqDRLJ4FL2Y*Ekki@3=3w8SI#ufYEGMd!8FzX87WpD7&`QX8f!^wF< zqfvLEWtEc$aY{RHeL6po#!weiJ5AC3LYmK$cFwY5V3;kFD@m&{A6)+SynkKZ{@Amo zv2L6i2tg}@t)m#j$4093?QlKjPgJUt4c!XmWzRcD+G)p1AcH6oW5d;6ξ0mvY-s zxkZON7M?VfFW0mwoNV z^F)nmf>jCXke;O>`BE?9HA0EzKDcEfSjheNcxweF=V1DJ3* zr7y6)%^GW6uRj;|UYCt33K;ca_xa6lCDs+z4S=QC0tj%m-k<4JY0{fTGjp}6TQ(b7WKm33lXEL$Nua#KEs z+!2%63Z}GV%N4jzZDAJRqE`p^n)X{VT-$?nob1~!T=UnV%^UtrOP%<{%)TJzPb>k1 z10WmLeOB2zOFx(0LWYK_U%IfWtzmNa-IL&@AIb&x`fFl?ujl0ja=b*z5eX+ZC>Oqw z1f^KXb8f8WXj#BQCR5sdB=F*-d)}Kuwbk-)XEQAQ`%<49I`dWf(`7z2uBI~b`3mOr z)4d&e>8@I=u7t+Aj}xEE)Au+;;NG{CcbbkV!l@6=_z_t&n)9V6xx)`bA9-D(-neU5 znOuo{`dB8^^LsNGV28`Kp_zeu7AhxvTl@RTG%Y>&Ah)K9?vbL-_|vtJb~IxGnGk^A@*_dy9;&>B~*?5vv&mo4Dot@A9n?mtv##$|}K$ zmg0wdt+j;1Ul%LzuK+VWi(QsmKg(rBdSl1Ud?>Y++=|kKX!rw zgV|hDXjT@4o(lY~b-MODp9bKh`mHCLO2e1@pRD22Q8;y)U;vrQ|uKE!jJe?qAFpa=1(CxwTx-IEw{kVWXSH zz}mQ+e=duizBQ_G@yT<)21ni;!@Cu(02V=rJ)yjh;sOP7L`f`KKGCyTR?!K=D+$eq zvsE?&Bbh?QUjMwmv+j>VOvBrr7~|i(5wg8L;v?5(GrOt8#Y@Taj`NzOfaBR$ch}ufPA_Des-AMwWfvRZIBPa*hhjpekW1%qz#4jD5GEy~kkPX^l(_{ofxULS& z?ciU-K-OM!IN&C*FcbXaC*T2m772b0sw!6=OkfHrQUpE(WBZWCEouI40Xib_qL;vm1d@n4jPwLzdL2KMGJ z^l;7yd&;vW)~h}B!nKd=k;*$_0Pokpli?R5-E4-hws#xG}GL!xmPHN}cAr zO~h_1beJyPa|32`6!$yr51_nHDO~vyyQW5?MS4RFVTFVKiv)-{tQ}Q`p@y&<^ffQ! zPRdI2Px-HW&*KN)$wqc~Am`?*^e0|Zv>U^ts`U4Z&QB#GB!*sbuzs0Tz2t82k`Y;vgHn%iHz z6Hlui<3QzK`of4&!*-R~;CFNQK!9#au&|hV8LXkgsxyQ$b|*ge*?-{=uJUs_9o9Zz zyWD&CMr5#uXC-b~fo*B{k!>dW^rb7cy|6&*R2T z5#%MArJlv@E966ieUF=F%(TKJQ1AWC#dd9O@^(990j~}Rf8G$aRi`;g_YJS3!ma78 zwmNBW%?BKyXejO)t_1$PGL?_+Z?*cu*6X=zQWwO{rGt1ZMSXufG(jGN!$ubv>Boum5|)SSMsB#W100XFKbt$$J>`Iebz9N zVlC6CCz=S48?b8wC6bDY(D&fwJ2M{*Hkz+P=+*V`*b%D>x@(Ddm^nJuQ6SyoulXE7 zW}8i~c=WJr$)(#?0f(Lk`x6CPN!-^rHU~2{Gyx$|NkmyC)w`w-`S%AO$&DtLcwlwD zw&dfU-rVcKg!e^W2G2^#yXeOf@qcXVXUgV39dMgjn9@H8hGpT@tX~H}oq200t-79V zUFj?-%)k$U#a|$vFqS@wr@{UTQpV z^PG$3H5+#~;T720`W3*8Xj#p$pj+r(ozAy@9R5KdVZ{!Huq@^@gK`)qq=01(a~Drk z7%hIgBtYq=w$9)ix8oVHXYKcE42{#qJ?S^fU3{Me8RQ4}$#_byFrA`{c!Y6wVwo8i za8IWtGKulW*g{*s?fPN`61}o$mm@-)MO9?CNZHV;}X{E>1U}Qw>!;T0fYuK?3`DinI|Z6p;QSO#HQu1y~{i)0I55ztO2*DG)L^3c}p z;ul*$+Z{D>wS)}N1uLa!WIu%Ujv$7VGkq+XQ89?T{{?i)IE?CViX6q41KmEKEW0CL zpTUu|BAt*12Epo3R+qCH{>uQysPdH*4r5)RNBO}YIrTjYUn0e4;Hv0Cr zVMXWbtHGuj81F+8t)m4)l1oXf92DjC^UuXWe-z4Nu6%eGk+{xhUY(8_y%f1r8a*|A z{`c?uECHG2{iAp-IdAbpI#4BA!st(hQ$;DA?v6d-i}kN=GT_I}R}pm806}^?AY3L9 zxn6b&I7)ZkKa(XqEt%O&6;s?Ey@iYC_T+ynn|GQhwF&+nVw(Cv@3luw{dOX<8{OI6 zlWwD(s`9B!a-|i@0ac%`6mYSe6 zFk!ymb&7ZP$UQ;eRSLS7SB1Hw(6`$z!2+_SfmO9~b{LAyt!CZdrkD< zr3abls@oUwa@_Qu7`JUOCfzS0eYlMOF?TjS?~ES|eOGv;@4r`2O!pKpPglA97`t#r z-53n)NVlMAdzgbl8DlPlWAk@*JTvH6BfLIeUb@9D{3fDg$Jhx(7BQr6Wn!#feSa?c z7_HUvP68u$6hb}7%xrhC=DhKcX+DMhBc)lM0|&@pmv9m%4`RMLXzp?swc*+1AIF=U z=jLi+{yVD9Dcdtm&LN-g@@$WoG}oaVi48RMkok0KuCc}N(TXZ|nmN<^o<$M_Hs2Zp z!QxNqa9VB~FRs5+bz%!2AX%YHTEjb!WQJ+Y?|FZ)qm;uYaaA8;cHY1BKGIJ!S*)$s zi(zV>8>$VO-hzyMjGbxvEh%!*auvnzK0!?e3%ec+ayAQdBg-&ZQm@I8GZGh%A+G#$nU>L{Qd(Vc9Cz}B zB|5+PkDUDzxx|tCnaYlG>7hgH?HI2|duLRrR|u{9sX8Z3tHBuDbzAFzo9V*8T!K1~cI?2D9&Ao1cH}snhkys$FN5DYwezCHK7t*Tp*!Ew@klLSmWeBm;1XPn zY5zD`n5y1hL^T1vnqDlqZmv(-LUUOx;c`dZ5*$NTVtEac^S{17xupoT@H#TOzO+A! zz3Q$en4#fU#JbR4$Rrl2!IAm4vHi=ie{9o2<{ie83y+;Vd=L}!3$$Dp$16~i+$6mZ zR{rd_l9(9?H?Y%W1t28()T`C*k7RFn6jXGGUh?eWLWlEHLYc?w#q zrAbs*>_Sno8r#96)NgKEjgXAZb#&TF4G9aBo)HbQ@Stlf@g;)jKvY?dLuBkWtgB^r z>r4>DrSM1Ow8FP4)?}sN~}nU5hTX1T2yNjn;Nl$RvUX&joPEJ5=9WJ)uL9> zmey>kqK(>GJNBqqF&k>PYRlj6?tk;X*YCMKH|OG9Jm+)1&(kN;$Uodqc?{ff6PiBW-# z6UrU-O9SCL680HOJ36CJjuax+l6B5L&DjRHX1wX)G+rz%)*db7CzGq^g0kHPL{f?^n_KsP zOG@t~VrxCdTe(siW~YS@mDDkjzmfab-Z_7$&JcegIah<2#-Yz&{z~rO;S|Yq&mWlfFuQ=%HK)H2NW1$E+Kd+uOu!qjyybVB+4%`sI{1xxuhxw{&0)+&eQ& z`Tb%9zET%}o85}9$NO@BY9+4JZ^)+d%I6JBNJ~$KM}B2!atiCFTy0@mp~lSZ=c-cw z*1yQvm{6i}BvTp_9zXbLi8M&@3#-vPS1ZX6{Si0XjNcpBO~+7(3k$`77hx)s^xEW1 zy^~$XOq^~|u6z9-!ooP!^uXshX!Vq|tL{p!>LEC`2KO;EmHTWJ>g9A)9k=toZoQBPk)4&c zhKDTAwtfoqh(#Wb(tUW5n=AUW|MdAsY72rp1x`I~i4;n@%C8gfx%&(`C62jUL zEF6Xh8v`s)oPhDnhT|nA@iS>xYa-rh)nUE+ST6Cw^RB4N4BKCd`I~-odRSAM*!@>h z>P5G{HB2_*?j%o+-v*>vnP%?{5*~44qcV>*6i|K7C-5vQWK2Jr=is(Of9~mC=iG)O zmc+3z#i^%M6c%&h!%wfi2SU>aZ_hKjMOt1aBHQvfNf)!vFTN`_JIo0MoLkBSXA!7v5puTmg{%JCTK&H9aXm}{Zx8t za{@m$yw2w$9^|Cjo$R|7-7d6FhSWkd{l3Yv9?sQJ=Kug;Ys+oFnCXx(JW2+J<<9lJ zq`0ebM*Z6Q`NGrC)A$n&y^w`1Ess}W=NTCPC7R^l?*Dvs$3UD4ADD!LDRNRjK((Zj zD?3U{D5wj*>k@FyzKe;$grf81CW@zhG^iBdh!EZ>PxTLOC1dsLcG76q-kV($eh#;y zq=O2)3^GAN4e?>=amatP9YuAs6bIEIiT|+1js~eFyIICfRh(RS1er26U|Oss9MKQ6 zEqbtW@U^>q*=2?izD$<1KP79<93WNJ4Qcu#X%m<%HhlceUMyBSWUw z^zz@8ILmjUs`X>I9Q#$F_QE}R1>$f9&AMdXyRU_;`a%ntus$y{#q1G*;cBF@zE0uK z%nPoqD}vL_5!~e=&0fC)mkAy5)5<29#tZn~q|E`|(abidafIRr6?o(oIpsTWQ=eWI6qX$F#HsavE z-AG0oUa17N6)#W(6g56W-H8oh5IonJoVRR8M@^xok;|jxq+9K7BfGCcLZFJbzm> zWlbvcgzkxV9+jw_S~MXI^TP1O>BQY}Gv>;nNXn_8A7*&xpnKZN6i{ew(9D5f)@qrx zMQz|vVSyEaC=4vaTqniOCvAMhE4d`RwgurTiC(UegF3_4W&l@1auM+@Toh`?4Zk%UL zw>&+=%|NjhpsxnZ9zI>n1qRk62S*{{jUKe!S%;uOhThbb1SbY*gVYt4i0$lqQXLN% z%l$W-Odg7U zH1Su;ej{3JcD0jnD(Z^Xq!=*m@VbTVn+u;8ajK^bbkUM=7ow0iCOXK;CSVWn4wna+ zl`J%ePh7bQF*j9uw*K)?ymo3f`~uMMn7W-ZWGk{3pDnMY7m9Y5CG0e9(t*|HqJKCP zwb3fJMW1E(FND?WG`Fz7E_~Obtrs7V{N=K^tn_9j9)jI1*gyQBIUQwIXQ9BRS&2vt zNO#a;M@XheSFDCK<<`1_N)YOsP*FT5*TNkt|X<{Id75m zRgBrZNrx||!~uF^L6l9I9#-&h5@ua!wk{l9HPcq^Ac}f8*N)4ac0y2r|7nL+2#LKqRX16o(qTgVbd{^Wd3Nv~; zZb1)B&7ZJsoQk^3@~tW81E@msHJ-;8QP*Y4cAf&h^Rz>2VhcwV8JAR4*$RbXVDWYu zg3^0V%ubf|jlT5W;)1j0Cv;26a4`cFV*(^7nhn?&W$6C9ptVyn;t}(5r_fMBfA-N| zua%9>#k$@md6mf@&s8XHYwh8F0vlz{WF9NC+k$fhGo9BZI>ju#g9@vnZ^Nj=?T;P4 zQxC{1jcQFtrgxCgRU)0N|5j1~34Xw!Ea6gc^PRZg>3~^~=rph(IbUHs1961e)bzX; z=ssKR`#5XoEMMjUAKiWf)856H{)Aip{4$(dQ|F*_&z;$}FBo<)j4RZs=BTS%eoMmE z)bpmT6pb7Qj#wFcZHWd>d$s7p;!w-3YSDZ-=<9Z>Oc*00Ec6Td>>FJWXrMq;7h+Qs z27F!k+{VpIRN{&I)a?m@!EF}3wxbrbNMEr<49sLbUS$ENs5Bdo# zg88GqI`~ymdGFkt^vJjwYkjw55qcNynbCE#bRLdb7?XqG_diYE!;H|?&OKP66--W8 z2_siVD1y+*2K|#E(@^l!%^6F7`44AM-0yB=OnPjmrW38Y-MGlcz z%GLU%5JTIdc`WN1ypRhom}{??iO~`V%po z;r?fE3xhKFZLs;w+xC?H*onSa?TLfBw5*X|(ZOs@4w$ZrM$?IQNDJ_a@`Rmyt zzvx7jH;rQME=Ws}lg*dCOYIDhMOz|gaoWSv~yPD|jHMf>GvLli2`l3}u>_PVB1*aOm2N*K4L{g|UTDraF zQcMSNnF%>_OOYra?2<1I8}4@^)Eap+=Z#Nk?PW=3r{3;vy`ZSI6LzGmhFaMT$y(pr zCSgohF?Xw;1FHpDj1uiXq@dJDY^#b>phQ`&#}w>}!&3{?n;0&lN(9CYM?IlcJ3zwI zZ1E-D-q=MHgg-5<4#z{vo_9)|ZqM}*VXi5g>g@e&G}ehq67i2~Ra9YLpP8%>XBrV<|i(u zJ-w2R!e2+a^|KN7n0i(s_J`ddwVzMr#s0M)obVj;{>9Kno$Lyc{Cf79tiusj&KYAH zU8d3+w^w$2%La02QIExP%5rot7Bn_p;_v*M=DYjNx3=BJH!AkuXcc!JJ*Pq5A`DhY zBr*Y>b96uo{qsbe=D2{Am+%HXIyt%3tl0-f-GwQloo+ekXW#Y+>mOzEb9tRPKowsA z6rAC*N~*@Qok_|mQSM}Vn@*I`7`QPhwl>N$M)e>2Nz+90c;5nJ*8k@siz&}WMHxUU zKY-tGlVrwZqEhc6b#jUN7GCiQP^oF&M3r`tXF(mmJPd&zOI8H!V9ST%j{cQ(|2dQH Le_r(czXAUb4~6YU diff --git a/OSHB Graphics/Thumbs.db b/OSHB Graphics/Thumbs.db deleted file mode 100644 index 677fdb23ab97f2c585ff40a28ea4def60c6589a4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10752 zcmeI$byOTnzBuqf1|J3wGFTwEy96IB*dPG{1P|^IJc4U*0we^21PH+)!8H)veS%wn zBsjt0_1wMpo!@(VcYk~K{`Jn;n)9K$tEIZDs+;QX$T0CzP9^m+=zk??5C-V_1{Z|& z|CFNv%P7AkA`l22SU{~`-`w2%tq=qP7XOp}54FG*aIKKv`w$bz!2ndBV*_vixBxr= zK7ar~2tf5W6v%-g03rsE07wC-^8cj&TP>gias_@~AO?^k$OXtfKz=|^{_BYX7%_j` z@qfP?{=wmITUdrn%tp{KfbGxe{OF)yC+a%>;g0`nLojMk1L=?Y#X!~|F`(v0AW`5a4lG%M_<-6)KsG?x zKTd2w{iu3T&(6PEKPRwn9O|Nh^hf=VfF&!SMp2N^Z+9;K+m)cMM&!5F_}kup{}*-t zsCxfMf4|4ymxusG%D=zZQGNDD`~54=Kk8k9dI$X3^G^nplLIIKw*Zs?DgZTr1^@%l z0#IE~4`2W=0&WAC0L*|p02Tl%fDOP7-~gcBEnGm(4d4Or0{8&@00Dp?KnNfVKpS_n+eS`sU`2BgP}vY{3s}LlmD2zkvoaq4(gFcXwJ&esRWNr9g2ZaJ_`kPW_m6BQm+d(s%Ekml_cla)Xf)BcoIpY-0B0 zTo~RZY<-5zp{^B?OAPeUY|OHbM4!9U6NllnQ!uH_t@RocVr=tEuA)}Tk7~IT2ee`&LdEa5 zJb*Gd8@0&K9JwFzI-{7qp`;dh4jKW_O(DJo$j5=AJNx2mnv~WdJw({sragE#8k_J2 zIZdjr39t7U=Az8|AClaC+Lr=V>i3h;wo4Hl%&D{wHd7$EPO51T#->RBlyIfn3g&Ez z>0%?&q|vtt!zR4$#@ZT&Bg-z)pOGTfRq6NT?tFF zbEis@+d{A23SF0q;rc-jpmS=&PHAOcZ>cyw7r7gmKd&E>YrI1EP<|A%OS-i}_v#jH z0Vl0pNHn)mv#0+QgvPkeVBf!rEdE18|9Z*C-h1PtS{%w*B}JCP-I~=Zp%msrcPUMo z``+Yo(&EV0$5QL(Fg_Uc%fkzF$xdT874-~|8kijHX0F9wv1#96_}cpP)(78Kr$f*6 zh%b>}B{b61S3>$@c7~}Z(mE_+Uw+XyfJ5`rw1Z}xHaQ~)U2H`nLi_}u{Md*K= zKR{e`_0d!LJkIL(SQ=@l{}lbv(?){1d&*S2*&e5BPxSQB!||6LA>u`v`(}2gh%*bY zV{v288(c@~m`w?38~vWjySXnhL=W$(}nO3;wclK%8d}wPxA~s ziczI#l$N}5QX1wZ5QFT$hDkJYiS>W)iCT$Qr-OI4z`{ID3ZA$;>&4|sl<}(i^ zP2OqbMB2pLz6tZl_d!c}*lN)(>)d1J-b=L!G!`n2^-c4P8jsnkG~C5A^4SUzEZw-s zpgZ!j?AXa?TzV$?=(Vd)?q1=5gL%WzgOs>uzAVLRK<%OX6!uGmON@&l(HktJ+FIg? zKdO!*6_PS&(XsDc-0a1Vny_w1W(dnG`x)2?M(nrjfT2>0!3$;u9cW@5!PhU{SNt2{ zt!*SZc_nQM%8?$i4NRPZ{5$0#q=5&yCubG1UxN_!dLqQK<)YCZDQ5B{qP!_jGrI-p zaD~olv5k!~%C>x4G8=t*WAjZo5{od-JyHI+cBJvBn|W`OKq?uwV!1r72z|UUMrcl0-}tvR$sko$*$t zCoYZlWTAvE>rg1B5QaVgv%7Qv@$0=pPZ;VteO*GuHLOZ2IPOX`@-H);l=h9J2y5;)OlWm6V?8Ienbk)7^r}$>GYFVALS(X3{V-w!Jgt*sYpZL#(d~s z%ujh9K4+w28_SAieQR#w2O@=vY#xSZ#xQ-<;yi{+xz&9P%X!^;)lX!2E!C)PV1A?$G->r5Vx3l_w=pph-wt&d}rBg6C=+0k-_>M%qFp;7OKC( z&$kq-z3%+}p{i2BF6(u8XMBW)tY|#-5q7xD&1_=GN+IQp!q-gGlP5tjL64re^zLxh zo2bj%I2w@s+KP5|THJm!zR#0o^SWJ<-64Z%#iEAB!Fx%mh?ntwFn?DKQRtVtbv&gx z9kDuLAL}em*w~Oo#78w}{5Mlb`Qvonj3nLOc62fMegAU_Db0-o-}E2HdjsBDTi*JW zso2#F1ZvTy74e!<1H@ZmNJz9G#f#0r4YLG2&HPY`I?1Qrwoh&@(WFk5Xp>DsikzJT zS2aoLl9TJ7Pz_`(TSR~D%5-7HuPH`$_t&aTc5BA4x`(%mNwmkHeI@xe;!;!3>XO(2 zO(`FjStb#s+luVbJWI{oCCr2XTnPlAf7T zYXnz`fYLSf)HzRvJ%C!YnedoKWkudHJe>T=$;I+rGsDQ?d$te-&1bwVvY#mMj+Kjk z=6yd`Y@B~;>fK0K7r5|nvvu3PGDz$^j`&d}kd;YoNd_rvz}S3p zZ(TA;CUJju^Qunh^6MU$4Oboe=Pdj71l8n?`{XxJ3+Tm(o;PCX2$MjI>StLo249v3 z-MyUw1fGUuqdngx(ak&BvQHT0vf3>-{MNz*nitt;zLXcdWnTM%A4bE^f%zgxRZ$aO zCT|{$TK@ObAM;ml^uL{>SXQ3{x9RS!KbgOxK3Sl@&0i&_Qc-jIe>Q(bXF$zg#k5`| zKUC4RpzL1s9J;&nxl|y6B-)aYqZOUedcl`}%V+>!am0L9(UM=N9F~X3B`H~-R^(G* zB9m(*fy34aH^am@eE2PcgXzaF)7$S2i(h98G74c|Eq8lnee=87y=c3j)4YHGrTWg- zS|^-cSvo_|kgZyOE3xn$;m%z)UbsW?Mj!8 z&>`_@$>jihfV1Meh5TpnCRlHlNHnFDntXqB5!X5%9Q$f%KeFIs5*7S*{$|BXPW>1! zbES`?v7AcvCbLOmdNKW_%hS&9zP}#-qIrEh2qUb87V{y=6y39V#yhRxoHJl_a-Bh& z_`m_WK4qahl9)P|2ALNl(wmDo%0{ie#sd=!qrJ^b0%DsV+=Zg;S-!K@y2idjVB3NA zXNG5Dmb?%+supQeK|LlfPp!Xf_Jzmq2dk9ige&a#_K-_)koqgK&wRaSju!=7}92z$8P#@KbU zGbKD^lgGA#^3MH4nY<}&ZG*>;EOcHK1sqmVD>dZqBJVTWD;$uxJwNJRZjyBBt(dVI zrF~;3G=Qyyaq>&#qz4{vLxVXqCn&95E&IK z)rD4m-A|B5+zu@W!d7bX`=SZrl<6+`j?0G{fPY~u|1o}fQQuWSnoUN07CCROuD}%?Gf=zDuSpq1N?e)Lcc&?K*K6o!|IpYn0lm_le3qSV0V-kB{?~XHW?W(Du{o%(>H!S zX*}UtP-Kw1uxYuH8md2${FdeU)b{bRTlzdiKelh;a}(4iBvA$pCKJwr2?d9Z?DM}1JPQQdBnawqBZD_P+$MEM& zOi6swmC=@z3%qjyPvqU2&$wGet+Ip%(2j}c64ByuK{elf-dhNr-!8jrE<`h%BAzT_ z2&=^S5p{+Bm~}hLzQmjSw*k7R7->mIGNo&DI}tW?Wo+;o#e_cpG?o1G%gc}3-jUY&%G zB8!(SFucKXN&m8@ol|iZUob@)+3=VWfhh@DS5p5vja6zyV>|6&llSg7CTnd8X0@v3 zgZSvFVvN-NrY*@po?#8UL%&+E!$1vCG2A#_)JBothdQh7LsR_+tlI^PY?M)*28TBN1& z2^^N1^I%n|LtN!t1-ONbxCg9121-~TtT?JXD(1$Eb8E~r4yN5@Kk!3NVIiBi7idax z7rtk~7AV3ZPkmivzJv(Y4;wi|TQO$66MdE5Zh1!KtaZDTJNaIFdxNguBu}?-YYB40 zrFFx-KNxQ2$}z0Wh{cUSaI57kk{9@cm2?|4oZN@5KSQp-NI#nHgin zmW}f6p7g_qR=SiS1-fo@vBk?UX*Y_jvxksLs>SjKyqa&C(^Eg(zidLdK4sH7x@_3L z)6#^e%XU~NI)dMqm7}Yw-hAzIm>y<*ns1t zVY?VJzAWjb-rk%3$>ViKZR=Tl4<541y39~?qO#C;#|48Kgk!RsWKEEJHMU0-YH{3* zPX~JJt(-GRtLc<8ivcafv zW8I+G0b=dv_1FD$93xnP@O^B8tlqVdwYSr8k&VmXCpssMGkZavsoWh0Nls7n0e2iq zrIm^7aB&PN-{YJkcnWSI7)!!sSHP~yd{>>(Ya?P_L02$N&Zu>!kN<^U09VH|k#cx=4o&`3n`}CMMppXDhn;5pVpY z&^|pGwApYyd?#BMl@&5n%jemP?I6KJAyg9fAQw}rO}K|VQ^K{ufCL-?OGxL!QQy0G zAVy|G5y17ikZ>ZqL{#sGxFwNWGkPPV=EqPWJ3spjTxE%9e2Ed846wKKZKD)%ll{tN zw>`xlA65u9h;dKomokAj661W%opi4ri>7SM~`7kw?df4RZqM%nQae6N=VcT)K5a zf|SfGGcVnvB4PPfgm*o<#i-e@4aHZGcEBKM*yw1b)ImO*q0`9^TtVJ?JDt7&?NMWBPoJpeIsv2 zEBLX!B{Cg~#_EZ@6O4(6i|hB(X1BTV>e`dOxiP?plmTaPAp0gWV5s?<g^H*VEA(|}iRXI7hgVEWH=&vy%AymK$j>WhwZ&Gx@$jMmaOFD{n@KL`;xiJab zTOMXKib>pvZ;dVc%RXM{?E=9a1)cq#V7Y~{epNfs+I9wORgVbx5m+^v*8@-3LsR`n(TL03P^&=`PEQ+ zr?)(ST!oWkHlfKRUz%ogu)wS;r^QBA&)Yes=H zdbf9bGDJQrs-VF}OZ`PhLy;_u5RQoayk$+87IP+u;WSbgFiYAWgJ0ZlaHALSeaua9 z4O)Ivy;}XaI{IO-qv!OT%bExawIhw87?Z;j8(KM$vL;2Y)x}Ii(9a6i`{qo;D;F2 zsW&#I;Dkw~A9M9sIl#l=HNBmdyw>DbaR<2~%T?TF`|VBpJaXxqq4;8%gmrS^Zzfqb zUybNa|M;=*I`c*R4i^V!o}Rc)f0S~bdg}Unv!8}SO1o1WJTaFBUvF)M99b4_TJmU{ zvt@_V<%$|7<~hob?R>3YOoybmy|oiED(w*n5#tM*j&sfms9w0kt$uL&;0+OS#TeYf z6P$g)0GDP~S*f_rvqd_yNk1ed+(cr2X{_#U!(G3p)<3+i0@q)XzX#u!Hyvrr51Tx| zXR!AXlApYu_Q%Z?n_$NwaQTcWmp=QwMTQQ8Sh6;!X|7G;q4B=Yd9vUd^dg1AFAMfE z0~W_8PDbZx_sE@ofeBKw_6)xs%sjTSqM63mdnbypchx4C`N<0bjaRa)g>b~n-k`7x zh@J@?-KE?2?z_m)4qhes9Em$~9el(Q)Y^0_Qe)4PyRo0*JX)h(MQGLAW#;Y`sS#z6 zq95Fu4aws3xMfA6Uo!eJ$mg+h*Eh=Y)#uo@!tLAr)(PFrhIN(pDGOO7;p=9TAA?d) zm&}`;ayK@&)lbhf(m`fL6%sz+qrs5A!YVMMVWTE5%_%wa{*oMfmw4dnDB{WLl$)-g zE;;$)t-})@KM8ixstyJj*=ha2NNC(-KaAn^cMr9Jb&AJ{EpU3H6f--il?wT37CtJZ zUdi$}2ppX>|3QZRMR6R5R=7-fU}TgFyYqzm0Od%dsF%)=vmB9!C{~MG<-r?lZPO2~ zPQv&~ngJyK;>`T_-f!}5K8nkdY7pnp%3kNS?1;%>(VivDB7pV}4ZGNtxtgAy7IW7q zd=2JuwZ-McV-TjS@czn4^9_;W9(_9|bY0We6ulC^JpMb9B7MfXsMX=4btui}lH8Kp zZreTYzwx=yuJJUp6T-779F5(qobNy8$%S~zU1o?f=pkM3)#P@}{AWWg9%F0@(r#ZQ z7+o_6Z`K_1vs55HT+_p^SxW7K;~u6gu%MmcYZdo6CyZANiATLBFb3(^m%4z@u^4H! z13?~rQ9?|R0&{e)ar5B778tUD(e0UHw8jpkBa=XGJwgbVPVDQRM%V|}NP zmk_n{uG+Sy6I&Nt|FIC^9wPwt-}f9YTItIb*i9d2}x8@+>5@;qT; zjI51I0T3Ql|LELwwUyV4ugbuyTF=dyke(l1+!;izHNA?03X)sNy>wFrLOWKv_PX6- zOUgO24&sBq@2Af()2`&cK$A$1O_$J@CG)Ij77bhWFI@6ha%H(Cjeq<}Ur>%}@ADVS z%ny#7riu~DM;@>xmWX8utzh57A#!6!pK@ekspy^Hs)+!(=R2(ST+yX9m7~p&#*an3 zG*aLBH+}{1Ra43=Ho)!WZ5B2;9BQBQxRUNC_z_@^;v>AkX*fs=W%E5F-HfL6LU=&JNFPU+`@pgJzJg7Ny8C5xQqRAgp<*nVdMZ!kNFM(eR31s z#;=A4MGy47xfeMyrM-1-NR$~_WJMiW+PK{Zf5d)$zK6d4^$1$tsi5EpJnj4M=IGZn zuzMMPqRp|*@$EgjT%UTYAVT<+%rYl_B`u@L{u)WoO2j&%5rRuptG4nfzv;XUzg+eT zE2&P^P_7K#dMuUhOZp5Q(?Ek4t|^WAa#;U%Vir!}{QdjQn_d#bQKrLaT5YWu=ZNd8 z@zW*Bem7-|Z+p#U{uxNpT{+yKlr_kbqQrvsOnzpzrg6&22=Xmv>~5R^UaUJ-@oW86 zzAHNCng=>NR$YEIKK7}XwP-}lRwMQU&j~zq@K!>X?H0Y86AGURd5t>2*5bS-oqY&O zK>Mb8k;WC%L{S&``&CSuq}w0J%P|frB;Kt$_eMneij=q%j7^8Ev*$QMyuK6&)fs(o zsH5vSB;W_LVN-+0yuN073Dg(hQ}8H=6WnpoM^LM?R`oJmQawvTpMG_zhDa&j4NeJm lLf-gxhg9C6Nn$ZGWe0NCqGq*!`9tO3n;QS;sQ*t3{1^ONFq!}W diff --git a/read/Image/Thumbs.db b/read/Image/Thumbs.db deleted file mode 100644 index 4b6fc3cd92f8bee324a611823a2a25c93557f959..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2560 zcmca`Uhu)fjZzO8(10BSGsD0CoD6J8;*3Bx2!nwD0|OI~0pkDr|NlQkkbwcn90fxt z1pWfu3W`4%9zsqZbt)A?Ac!F!2um0g7+e|hfOHW4kQmmK@?nMbM|CH8}Y0aT8I;)Df=LFK?G7&##z0gZb(XnuJ4{lDb-#lM3( z1N_{1xum#&(!8D?E*8o z|0J>k`6oPG978PplmGnxzu)%o{%#(J1B^$s8zLBvG&WAwmp7Qvt=%vqu;&ch>4Qxk z4IlsC=YHql8sE^>*C4W(dBrnMhqJ8-j3x|52jo2D9X4^wK4CQVZ`)+Zc%k9H`n=p9 z{EHkVw0ukwnl2?UZ0Qs{r6RRJ!huhsf?2|mu|t1e<4YHT8Ga6iUzs}kn^`9^Xh=CK z%5z-V!>H84)$pnBfW|-eEi5uuUofON*spV8VsfZVSJ=w#Z*`F0LGQ?PKMyXKoR=vl gmNT=lG1SQ?elR;90dC=Vy85}Sb4q9e`fr#30GWMB8vp*8o z|0J>k`6oPG978PplmGnxzu)%o{%#(J1B^$s8zLBvG&WAwmp7Qvt=%vqu;&ch>4Qxk z4IlsC=YHql8sE^>*C4W(dBrnMhqJ8-j3x|52jo2D9X4^wK4CQVZ`)+Zc%k9H`n=p9 z{EHkVw0ukwnl2?UZ0Qs{r6RRJ!huhsf?2|mu|t1e<4YHT8Ga6iUzs}kn^`9^Xh=CK z%5z-V!>H84)$pnBfW|-eEi5uuUofON*spV8VsfZVSJ=w#Z*`F0LGQ?PKMyXKoR=vl gmNT=lG1SQ?elR;90dC=Vy85}Sb4q9e`fr#30Ow9gC;$Ke From 22484c0efeb54b380469e6adf36a37c61cc7bebd Mon Sep 17 00:00:00 2001 From: Jesse Griffin Date: Tue, 10 Nov 2026 22:13:51 -0500 Subject: [PATCH 2/2] Close gaps in the data validator found in review Address the six findings from the review of 6623bd0, each reproduced first by mutating a copy of the dataset: 1. wlc/VerseMap.xml was never parsed, so a malformed copy passed. Every tracked XML file not already read by a specific check is now parsed for well-formedness. 2. A malformed file in the base revision failed the run. The base is now read quietly; the summary names what could not be compared, and any error while producing the summary is reported there instead of failing the run. 3. A outside any verse skipped all checks. It is now an error. 4. The id, morph, n and verse-id patterns used "$", which accepts a trailing newline, so a six-character id passed. They now use fullmatch(). 5. Deleting MAPM/MAPM.xml passed. A missing file is now an error. 6. A verse milestone with both sID and eID, or with neither, was not reported. Each milestone must now carry exactly one. Also from the review: - The change summary now lists verses whose words are the same but in a different order, which the id-keyed comparison could not show. - parse() and _quiet_parse() are merged into one function with a quiet flag, so the two comparison paths share one error policy. Co-Authored-By: Claude Fable 5.2 --- .github/scripts/validate_data.py | 214 ++++++++++++++++++++----------- 1 file changed, 136 insertions(+), 78 deletions(-) diff --git a/.github/scripts/validate_data.py b/.github/scripts/validate_data.py index 3b9bfb9..749ec85 100644 --- a/.github/scripts/validate_data.py +++ b/.github/scripts/validate_data.py @@ -5,16 +5,16 @@ Checks (any failure exits 1): * every tracked .xml file is well-formed - * wlc/: 39 books, the expected verse inventory, and every carries a - lemma, a morph code and a unique five-character id - * MAPM/MAPM.xml: verse milestones are balanced and its verse inventory - matches wlc/ (apart from the verses MAM's text does not have) + * wlc/: 39 books, the expected verse inventory, every is inside a + verse and carries a lemma, a morph code and a unique five-character id + * MAPM/MAPM.xml: present, verse milestones are balanced, and its verse + inventory matches wlc/ (apart from the verses MAM's text does not have) * no operating-system junk files are tracked With --base (a checkout of the revision being merged into) it also writes a summary of which words and verses changed, to stdout and, on GitHub Actions, -to the job summary. The summary never fails the build; it is there to make -review easier. +to the job summary. The summary never fails the build: problems in the base +revision, or in producing the summary, are reported in the summary only. Standard library only. """ @@ -33,13 +33,16 @@ # Verses that BHS numbers but MAM's text does not have. MAPM_ABSENT = {"Josh.21.36", "Josh.21.37"} JUNK = re.compile(r"(^|/)(Thumbs\.db|\.DS_Store|desktop\.ini)$", re.I) -MORPH = re.compile(r"^[HA][A-Za-z0-9]+(/[A-Za-z0-9]+)*$") -STRUCT = re.compile(r"^\d+(\.\d+)*$") -WORD_ID = re.compile(r"^[0-9]{2}[A-Za-z0-9]{3}$") -VERSE_ID = re.compile(r"^[1-2]?[A-Za-z]+\.\d+\.\d+$") +# These are applied with fullmatch(): "$" alone would accept a trailing newline. +MORPH = re.compile(r"[HA][A-Za-z0-9]+(/[A-Za-z0-9]+)*") +STRUCT = re.compile(r"\d+(\.\d+)*") +WORD_ID = re.compile(r"[0-9]{2}[A-Za-z0-9]{3}") +VERSE_ID = re.compile(r"[1-2]?[A-Za-z]+\.\d+\.\d+") +WORD_FIELDS = ("text", "lemma", "morph", "n", "type", "verse") MAX_ROWS = 200 errors = [] +parsed = set() # absolute paths already checked for well-formedness def error(msg): @@ -59,13 +62,22 @@ def tracked_files(root): return found -def parse(path): +def parse(path, label=None, quiet=False): + """Parse an XML file. Returns its root, or None if it cannot be parsed. + + A failure is a validation error unless quiet is set, which is how the + base revision is read: its problems must not fail the current one. + """ + if not quiet: + parsed.add(os.path.abspath(path)) try: return ET.parse(path).getroot() except ET.ParseError as exc: - error(f"{path}: not well-formed XML: {exc}") + if not quiet: + error(f"{label or path}: not well-formed XML: {exc}") except OSError as exc: - error(f"{path}: cannot read: {exc}") + if not quiet: + error(f"{label or path}: cannot read: {exc}") return None @@ -75,9 +87,17 @@ def tag(el): # ---------------------------------------------------------------- wlc/ +Wlc = collections.namedtuple("Wlc", "words verses order unreadable") + + def load_wlc(root, check): - """Return ({word id: record}, [verse ids]) for the wlc/ books under root.""" - words, verses = {}, [] + """Read the wlc/ books under root. + + words: {word id: record}; verses: [verse ids]; order: {verse id: [word + ids]}; unreadable: book files that could not be parsed. With check set, + problems are recorded as validation errors. + """ + words, verses, order, unreadable = {}, [], {}, [] wlc = os.path.join(root, "wlc") books = sorted(f for f in os.listdir(wlc) if f.endswith(".xml") and f != "VerseMap.xml") if os.path.isdir(wlc) else [] @@ -85,53 +105,70 @@ def load_wlc(root, check): error(f"wlc/: expected {WLC_BOOKS} book files, found {len(books)}") for name in books: rel = f"wlc/{name}" - doc = parse(os.path.join(wlc, name)) + doc = parse(os.path.join(wlc, name), label=rel, quiet=not check) if doc is None: + unreadable.append(rel) continue book = name[:-4] + in_verse = set() for verse in doc.iter(NS + "verse"): vid = verse.get("osisID", "") verses.append(vid) - if check and (not VERSE_ID.match(vid) or vid.split(".")[0] != book): + ids = order.setdefault(vid, []) + if check and (not VERSE_ID.fullmatch(vid) or vid.split(".")[0] != book): error(f"{rel}: unexpected verse osisID {vid!r}") for w in verse.iter(NS + "w"): + in_verse.add(w) wid = w.get("id") - rec = {"verse": vid, "text": "".join(w.itertext()), + rec = {"verse": vid, "book": rel, "text": "".join(w.itertext()), "lemma": w.get("lemma"), "morph": w.get("morph"), "n": w.get("n"), "type": w.get("type")} if check: where = f"{rel} {vid} word {wid!r}" - if wid is None or not WORD_ID.match(wid): + if wid is None or not WORD_ID.fullmatch(wid): error(f"{where}: missing or malformed id") elif wid in words: error(f"{where}: duplicate id (also in {words[wid]['verse']})") if not rec["lemma"]: error(f"{where}: missing lemma") - if not rec["morph"] or not MORPH.match(rec["morph"]): + if not rec["morph"] or not MORPH.fullmatch(rec["morph"]): error(f"{where}: missing or malformed morph {rec['morph']!r}") - if rec["n"] is not None and not STRUCT.match(rec["n"]): + if rec["n"] is not None and not STRUCT.fullmatch(rec["n"]): error(f"{where}: malformed n {rec['n']!r}") if not rec["text"].strip(): error(f"{where}: empty word") if wid is not None: + ids.append(wid) words.setdefault(wid, rec) + if check: + for w in doc.iter(NS + "w"): + if w not in in_verse: + text = "".join(w.itertext()).strip() + error(f"{rel}: word outside any verse (id {w.get('id')!r}, text {text[:20]!r})") if check: if len(verses) != WLC_VERSES: error(f"wlc/: expected {WLC_VERSES} verses, found {len(verses)}") if len(set(verses)) != len(verses): dup = [v for v, c in collections.Counter(verses).items() if c > 1] error(f"wlc/: duplicate verse ids: {dup[:5]}") - return words, verses + return Wlc(words, verses, order, unreadable) # ---------------------------------------------------------------- MAPM/ def load_mapm(root, check): - """Return {verse id: text} for MAPM/MAPM.xml, or None if absent/unparseable.""" + """Return {verse id: text} for MAPM/MAPM.xml, or None if absent or unparseable. + + With check set, problems (including the file being absent) are recorded + as validation errors. + """ + rel = "MAPM/MAPM.xml" path = os.path.join(root, "MAPM", "MAPM.xml") if not os.path.exists(path): + if check: + error(f"{rel}: file is missing") return None - doc = parse(path) if check else _quiet_parse(path) + doc = parse(path, label=rel, quiet=not check) if doc is None: return None verses, current, skip = collections.OrderedDict(), None, 0 @@ -140,105 +177,119 @@ def walk(el): nonlocal current, skip name = tag(el) if name == "verse": - if el.get("sID"): + start, end = el.get("sID"), el.get("eID") + if check and bool(start) == bool(end): + which = "both sID and eID" if start else "neither sID nor eID" + error(f"{rel}: verse milestone {start or el.get('osisID')!r} has {which}") + if start: if check and current is not None: - error(f"MAPM/MAPM.xml: verse {el.get('sID')} starts before {current} ends") - current = el.get("sID") + error(f"{rel}: verse {start} starts before {current} ends") + current = start if check and current in verses: - error(f"MAPM/MAPM.xml: duplicate verse {current}") + error(f"{rel}: duplicate verse {current}") verses[current] = [] - elif el.get("eID"): - if check and el.get("eID") != current: - error(f"MAPM/MAPM.xml: verse end {el.get('eID')} does not match open verse {current}") + elif end: + if check and end != current: + error(f"{rel}: verse end {end} does not match open verse {current}") current = None muted = name in ("note", "title", "header") skip += muted if el.text and not skip: - _text(el.text) + text(el.text) for child in el: walk(child) if child.tail and not skip: - _text(child.tail) + text(child.tail) skip -= muted - def _text(s): + def text(s): if current is not None: verses[current].append(s) elif check and s.strip(): - error(f"MAPM/MAPM.xml: text outside any verse: {s.strip()[:30]!r}") + error(f"{rel}: text outside any verse: {s.strip()[:30]!r}") walk(doc) if check and current is not None: - error(f"MAPM/MAPM.xml: verse {current} is never closed") + error(f"{rel}: verse {current} is never closed") return {v: re.sub(r"\s+", " ", "".join(parts)).strip() for v, parts in verses.items()} -def _quiet_parse(path): - try: - return ET.parse(path).getroot() - except (ET.ParseError, OSError): - return None - - def letters(s): return "".join(c for c in s if "א" <= c <= "ת") # ---------------------------------------------------------------- report -def report(root, base, words, mapm): - out = ["## Data change summary", ""] - base_words, _ = load_wlc(base, check=False) - added = [i for i in words if i not in base_words] - removed = [i for i in base_words if i not in words] +def clip(items, limit): + return ", ".join(items[:limit]) + (" …" if len(items) > limit else "") + + +def report(base, wlc, mapm): + """Markdown summary of what differs from the base revision. + + The base is read quietly: nothing here adds to the validation errors. + """ + out = ["## Data change summary", "", "### wlc/"] + old = load_wlc(base, check=False) + skipped = set(old.unreadable) + if skipped: + out.append(f"- not compared, because the base revision's copy could not be read: " + f"{clip(sorted(skipped), 40)}") + words = {i: r for i, r in wlc.words.items() if r["book"] not in skipped} + added = [i for i in words if i not in old.words] + removed = [i for i in old.words if i not in wlc.words] changed = [] for wid, rec in words.items(): - old = base_words.get(wid) - if old is None: + before = old.words.get(wid) + if before is None: continue - fields = [f for f in ("text", "lemma", "morph", "n", "type", "verse") if rec[f] != old[f]] + fields = [f for f in WORD_FIELDS if rec[f] != before[f]] if fields: - changed.append((wid, rec, old, fields)) - out.append("### wlc/") - if not (added or removed or changed): - out.append("No word-level changes.") + changed.append((wid, rec, before, fields)) + reordered = [v for v, ids in wlc.order.items() + if v in old.order and ids != old.order[v] and sorted(ids) == sorted(old.order[v])] + if not (added or removed or changed or reordered): + out.append("No word-level changes." if not skipped else "No word-level changes in the books compared.") else: per_field = collections.Counter(f for _, _, _, fs in changed for f in fs) out.append(f"- words changed: **{len(changed)}** " f"({', '.join(f'{f}: {n}' for f, n in per_field.most_common()) or 'none'})") out.append(f"- word ids added: **{len(added)}**, removed: **{len(removed)}**") + out.append(f"- verses whose words are the same but in a different order: **{len(reordered)}**" + + (f" — {clip(reordered, 60)}" if reordered else "")) if changed: out += ["", "| verse | id | field | before | after |", "|---|---|---|---|---|"] - rows = [(rec["verse"], wid, f, old[f], rec[f]) - for wid, rec, old, fs in changed for f in fs] + rows = [(rec["verse"], wid, f, before[f], rec[f]) + for wid, rec, before, fs in changed for f in fs] for v, wid, f, a, b in rows[:MAX_ROWS]: out.append(f"| {v} | `{wid}` | {f} | {a if a is not None else '(none)'} " f"| {b if b is not None else '(none)'} |") if len(rows) > MAX_ROWS: out.append(f"| … | | | {len(rows) - MAX_ROWS} more rows not shown | |") - for label, ids, src in (("added", added, words), ("removed", removed, base_words)): + out.append("") + for label, ids, src in (("added", added, words), ("removed", removed, old.words)): if ids: - shown = ", ".join(f"`{i}` ({src[i]['verse']})" for i in ids[:30]) - out.append(f"- {label}: {shown}{' …' if len(ids) > 30 else ''}") + shown = clip(["`%s` (%s)" % (i, src[i]["verse"]) for i in ids], 30) + out.append(f"- {label}: {shown}") out += ["", "### MAPM/"] - base_mapm = load_mapm(base, check=False) - if mapm is None or base_mapm is None: + old_mapm = load_mapm(base, check=False) + if mapm is None or old_mapm is None: out.append("MAPM/MAPM.xml is missing or unreadable on one side; not compared.") - elif mapm == base_mapm: + elif mapm == old_mapm: out.append("No verse text changes.") else: - shared = [v for v in mapm if v in base_mapm] - differ = [v for v in shared if mapm[v] != base_mapm[v]] + shared = [v for v in mapm if v in old_mapm] + differ = [v for v in shared if mapm[v] != old_mapm[v]] marks_only = [v for v in differ if unicodedata.normalize("NFD", mapm[v]).replace(" ", "") - == unicodedata.normalize("NFD", base_mapm[v]).replace(" ", "")] - cons = [v for v in differ if letters(mapm[v]) != letters(base_mapm[v])] - out.append(f"- verses only in this revision: **{len(set(mapm) - set(base_mapm))}**, " - f"only in base: **{len(set(base_mapm) - set(mapm))}**") + == unicodedata.normalize("NFD", old_mapm[v]).replace(" ", "")] + cons = [v for v in differ if letters(mapm[v]) != letters(old_mapm[v])] + out.append(f"- verses only in this revision: **{len(set(mapm) - set(old_mapm))}**, " + f"only in base: **{len(set(old_mapm) - set(mapm))}**") out.append(f"- shared verses whose text differs: **{len(differ)}** of {len(shared)} " f"({len(marks_only)} differ only in spacing or mark order)") out.append(f"- of those, verses whose consonants differ: **{len(cons)}**" - + (f" — {', '.join(cons[:60])}{' …' if len(cons) > 60 else ''}" if cons else "")) + + (f" — {clip(cons, 60)}" if cons else "")) return "\n".join(out) + "\n" @@ -255,14 +306,11 @@ def main(): for f in files: if JUNK.search(f): error(f"{f}: operating-system junk file is tracked; please remove it") - for f in files: - if f.endswith(".xml") and not f.startswith("wlc/") and f != "MAPM/MAPM.xml": - parse(os.path.join(root, f)) - words, verses = load_wlc(root, check=True) + wlc = load_wlc(root, check=True) mapm = load_mapm(root, check=True) - if mapm is not None and verses: - expected = set(verses) - MAPM_ABSENT + if mapm is not None and wlc.verses: + expected = set(wlc.verses) - MAPM_ABSENT got = set(mapm) if got != expected: extra, missing = sorted(got - expected), sorted(expected - got) @@ -270,12 +318,22 @@ def main(): f"(unexpected: {extra[:8]}{' …' if len(extra) > 8 else ''}; " f"missing: {missing[:8]}{' …' if len(missing) > 8 else ''})") - print(f"wlc/: {len(verses)} verses, {len(words)} words") + # Every other tracked XML file, whatever it is, must at least be well-formed. + for f in files: + path = os.path.join(root, f) + if f.lower().endswith(".xml") and os.path.abspath(path) not in parsed: + if os.path.exists(path): + parse(path, label=f) + + print(f"wlc/: {len(wlc.verses)} verses, {len(wlc.words)} words") if mapm is not None: print(f"MAPM/: {len(mapm)} verses") if args.base: - summary = report(root, args.base, words, mapm) + try: + summary = report(args.base, wlc, mapm) + except Exception as exc: # the summary is a convenience and must never fail the run + summary = f"## Data change summary\n\nThe summary could not be produced: {exc!r}\n" print("\n" + summary) target = os.environ.get("GITHUB_STEP_SUMMARY") if target: