#!/usr/bin/env python3
"""Which words are in the review, and which aren't?

Fact-Checked Science, November 2026, for the paper check /biosphere-tipping-warning/ ("Our analysis").
Standard library only. It reads text files as data.

Rule, stated before the run: count each term case-insensitively over the whole text, with words broken at line
ends closed up and the publisher's page footer removed. Hits before the line that starts the reference list are "body";
the rest are "references". Every hit prints with its page. Text inside image figures and the Supporting Information
aren't covered.

Run it
  pdftotext -enc UTF-8 Abrams-2026-RevGeophys.pdf paper.txt       (the paper's PDF, from its DOI, CC BY 4.0)
  python3 word_search.py paper.txt [other-review.txt ...]
Pages are split at pdftotext's form feeds. Each other review is a plain-text copy, and only its term counts print.
"""
import re
import sys

sys.stdout.reconfigure(encoding="utf-8")
FOOT = re.compile(r"19449208, 2026, 4, Downloaded from.*?Creative Commons License", re.S)
HY = "‐"

TERMS = [
    ("solar", r"solar"), ("sun", r"\bsun\b"), ("sunspot", r"sunspot"), ("sunlight", r"sunlight"),
    ("irradiance", r"irradiance"), ("insolation", r"insolation"), ("geomagnet*", r"geomagnet"),
    ("magnet*", r"magnet"), ("pole / poles", r"\bpoles?\b"), ("pole shift", r"pole.{0,2}shift"), ("polar", r"\bpolar"),
    ("cosmic", r"cosmic"), ("galactic", r"galactic"), ("aurora", r"aurora"), ("space weather", r"space.{0,2}weather"),
    ("ionosph*", r"ionosph"), ("excursion", r"excursion"), ("reversal", r"reversal"),
    ("volcan*", r"volcan"), ("eruption", r"eruption"), ("bolide", r"bolide"), ("asteroid", r"asteroid"),
    ("extra-terrestrial", r"extra.{0,2}terrestrial"), ("subsurface", r"subsurface"), ("orbital", r"orbital"),
    ("Milankovitch", r"milankovitch"), ("ENSO", r"\bENSO\b"), ("El Nino", r"el ni.o"),
    ("natural variab*", r"natural variab"), ("internal variab*", r"internal.{0,2}variab"), ("natural signal/forcing", r"natural (signal|forc)"),
    ("non-anthropogenic", r"non.{0,2}anthropogenic"), ("anthropogenic", r"anthropogenic"), ("ozone", r"\bozone\b"),
    ("UV", r"\bUV\b"), ("ultraviolet", r"ultraviolet"), ("radiation", r"radiation"), ("tectonic", r"tectonic"), ("earthquake", r"earthquake"),
    ("mainstream", r"mainstream"), ("false positive/alarm/signal", r"false (positive|alarm|signal|negative)"),
]
DRIVERS = [
    ("climate change / warming", r"climate change|warming"), ("land use / land-use", r"land.{0,1}use"), ("deforestation", r"deforestation"),
    ("fire", r"\bfires?\b"), ("drought", r"drought"), ("grazing / overgrazing / herbivory", r"grazing|herbivor"),
    ("insect / bark beetle", r"insect|beetle"), ("disease", r"disease"), ("invasive species", r"invasive"),
    ("aquaculture / coastal development", r"aquaculture|coastal development"), ("sea-level rise", r"sea.{0,1}level"),
    ("human activities / anthropogenic", r"human activit|anthropogenic"),
]
# Is each paper in the review's reference list? Matched on words of its title or its authors' names.
CHECKS = [
    ("Brook et al. 2013", r"have planetary tipping points"), ("Montoya et al. 2018", r"Implausible science"), ("Hillebrand et al. 2020", r"Hillebrand"),
    ("Kopp et al. 2025", r"Kopp, R"), ("Ben-Yami et al. 2024", r"Ben.{0,1}Yami"), ("Hastings & Wysham 2010", r"Wysham"),
    ("Boettiger & Hastings 2012, limits to detection", r"Boettiger, C"), ("Boettiger & Hastings 2012, prosecutor's fallacy", r"prosecutor"),
    ("Feng et al. 2021", r"Feng, Y"), ("Higgins & Scheiter 2012", r"Higgins, S"), ("Nitzbon et al. 2024", r"Nitzbon, J"),
    ("Runge et al. 2025", r"Runge, "), ("T. Smith et al. 2023, multi-instrument", r"multi.{0,1}instrument"),
    ("Lenton & Williams 2013", r"Lenton, T\. M\., & Williams"), ("Hughes et al. 2013", r"Hughes, T"),
    ("Tao et al. 2023", r"Little evidence"),
]


def clean(s):
    s = FOOT.sub(" ", s)
    s = re.sub(r"\s+", " ", s)
    return re.sub(HY + r" (?=\w)", HY, s)


def pages_of(text):
    marked = re.split(r"\n===== PDF page (\d+) =====\n", text)
    if len(marked) > 1:
        return [(int(marked[i]), marked[i + 1]) for i in range(1, len(marked), 2)]
    return list(enumerate(text.split("\f"), 1))


def search(path):
    docs, in_refs = [], False
    for n, p in pages_of(open(path, encoding="utf-8", errors="replace").read()):
        c = clean(p)
        if not in_refs and "References Abbott, K. C." in c:
            i = c.index("References Abbott, K. C.")
            docs += [(n, "body", c[:i]), (n, "refs", c[i:])]
            in_refs = True
        else:
            docs.append((n, "refs" if in_refs else "body", c))
    return docs


def main():
    if len(sys.argv) < 2:
        sys.exit("Usage: python3 word_search.py paper.txt [other-review.txt ...]")
    docs = search(sys.argv[1])
    print("Passages below come from Abrams et al. (2026), Reviews of Geophysics 64(4), e2025RG000900,")
    print("https://doi.org/10.1029/2025RG000900, CC BY 4.0. Hyphens and spacing as pdftotext gives them.")
    if len(sys.argv) > 2:
        print("The OTHER REVIEW lines hold short passages from Bathiany et al., Loriani et al. and J. F. Abrams et al., each CC BY 4.0.")
    print()
    print("THE REVIEW: terms\n")
    for name, pat in TERMS:
        hits = []
        for n, kind, c in docs:
            for m in re.finditer(pat, c, flags=re.I):
                hits.append((n, kind, c[max(0, m.start() - 110):min(len(c), m.end() + 110)]))
        print("### %s: %d in body, %d in references" % (name, sum(h[1] == "body" for h in hits), sum(h[1] == "refs" for h in hits)))
        for n, kind, ctx in hits:
            print("  p.%d [%s]: ...%s..." % (n, kind, ctx))
        print()
    print("THE REVIEW: drivers named in the body\n")
    for name, pat in DRIVERS:
        found = [(n, len(re.findall(pat, c, flags=re.I))) for n, kind, c in docs if kind == "body"]
        found = [(n, k) for n, k in found if k]
        print("%-36s %4d hits on pp. %s" % (name, sum(k for _, k in found), ", ".join(str(n) for n, _ in found)))
    print("\nTHE REVIEW: in its reference list?\n")
    refs = " ".join(c for _, kind, c in docs if kind == "refs")
    for name, pat in CHECKS:
        print("%-50s %s" % (name, "yes" if re.search(pat, refs) else "no"))
    for other in sys.argv[2:]:
        text = open(other, encoding="utf-8", errors="replace").read()
        print("\nOTHER REVIEW: %s" % other.replace("\\", "/").split("/")[-2:][0])
        for name, pat in TERMS[:18]:
            k = len(re.findall(pat, text, flags=re.I))
            if name in ("solar", "geomagnet*", "magnet*", "pole shift", "cosmic", "space weather", "aurora", "volcan*"):
                print("  %-14s %d" % (name, k))
        for m in list(re.finditer(r"solar", text, flags=re.I))[:4]:
            print("    solar: ...%s..." % re.sub(r"\s+", " ", text[max(0, m.start() - 60):m.end() + 40]))


if __name__ == "__main__":
    main()