diff --git a/README.md b/README.md index d2badff..9f3b432 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,15 @@ replacement. it names. It ends with a comment naming the git commit of the bot that wrote it, such as ``. It's saved on every run, so its history is a log of the runs. +7. **Adds a line to the report history,** `User:ParamBot/ReportHistory`, on + live and report-only runs, newest first: when the run started and ended, + how long it took, whether it was live, a BRFA trial or reporting only, how + many categories had pages, and how many pages it checked, edited and left + for review. The last column says how the run ended (finished, finished + early and why, stopped by an error, or aborted by the Run page, saying + when it was changed and by whom), with a permanent link to the report it + saved. The bot creates the page if need be, and keeps the newest 200 lines; + anything above the table can be edited by hand. ## Safeguards @@ -133,7 +142,10 @@ is due to expire, or if `User:ParamBot/Rules/Instructions`, lets a report-only run update the report, but still stops any run that would edit articles. After that it makes no more wiki edits, not even the report, which is written to a - local file instead. + local file instead. The one exception is its line on + `User:ParamBot/ReportHistory`, in the bot's own userspace, which says the + run was aborted, when the Run page was changed and by whom. A run that the + Run page stops before it starts adds no line. - **Large runs are flagged.** There's no limit on edits per run, but a run of more than 500 edits (`large_run`) gets a note on the report, so a rule that catches more than it should is noticed. `max_edits`, or `--max-edits N`, @@ -361,7 +373,7 @@ Options for `run`: | `--template NAME` | all | Only use this template's rules. Can be repeated. | | `--page TITLE` | | Only check this page, skipping the categories. Can be repeated. | | `--any-namespace` | off | With `--page`, allow non-articles such as sandboxes. Dry runs only. | -| `--report-page`, `--run-page` | `[pages] report`, `[pages] run` | Pages to use instead. | +| `--report-page`, `--run-page`, `--history-page` | `[pages] report`, `[pages] run`, `[pages] history` | Pages to use instead. | | `--out-dir DIR` | `[output] dir` | Where dry runs, and live runs that can't save the report, write their files. | Without `--dry-run`, `--report-only` or `--live`, a run does what `mode` in @@ -391,7 +403,7 @@ wiki is the English Wikipedia unless `--lang` or `--family` says otherwise. | Setting | Default | Meaning | |---|---|---| | `mode` | `"dry-run"` | What a plain `parambot run` does: `"dry-run"`, `"report-only"` or `"live"`. | -| `[pages]` `rules`, `report`, `run`, `instructions`, `faq`, `header`, `link_rule` | `User:/Rules`, `/Report` and so on; `instructions` is the rules page's `/Instructions` | The bot's pages. The report page can't be one of the others. | +| `[pages]` `rules`, `report`, `run`, `history`, `instructions`, `faq`, `header`, `link_rule` | `User:/Rules`, `/Report`, `/ReportHistory` and so on; `instructions` is the rules page's `/Instructions` | The bot's pages. The report page can't be one of the others. The history page must be a page of its own under `User:/`. | | `[limits]` `cooldown_days` | `30` | Don't edit a page the bot edited this recently. `0` turns it off. | | `[limits]` `max_hours` | `20` | Stop starting new work after this many hours, so a run is done before the next starts. The Toolforge job overrides it with `--max-hours 5`. `0` means no limit. | | `[limits]` `max_edits` | `0` | Stop after this many edits. `0` means no limit. | @@ -485,7 +497,8 @@ job then does what the settings say.) A run goes `cli` → `bot`, which checks the bot's pages (`botpages`), finds and reads the rules pages (`rulespages`, `rules`), checks the rules against the templates (`prepare`), then applies the active ones to each article -(`fixer`) and writes the report (`report`). Only `wiki` talks to the wiki. +(`fixer`) and writes the report (`report`), and a line for the run on the +report history (`history`). Only `wiki` talks to the wiki. | Path | Contents | |---|---| @@ -502,6 +515,7 @@ the templates (`prepare`), then applies the active ones to each article | `parambot/templatescan.py` | Reading a template's own parameter checks, and wrapper templates; `scaffold`. | | `parambot/luapattern.py` | Lua patterns, as used in templates, translated to Python regexes. | | `parambot/report.py` | The report page. | +| `parambot/history.py` | The report history page: a line for each run. | | `parambot/messages.py` | Everything the bot says on the report, in edit summaries and when it stops. | | `parambot/wiki.py` | Everything the bot asks of the wiki, through Pywikibot. | | `parambot/wikitext.py` | Small wikitext helpers. | diff --git a/deploy/parambot.example.toml b/deploy/parambot.example.toml index d9c6c45..d84ec71 100644 --- a/deploy/parambot.example.toml +++ b/deploy/parambot.example.toml @@ -27,6 +27,10 @@ # if it's a subpage of User:. # report = "User:ParamBot/Report" # run = "User:ParamBot/Run" +# Live and report-only runs add a line here for each run, newest first: when +# it ran, what it did, and how it ended, including when the Run page stopped +# it. It must be a subpage of User:; the bot creates it if need be. +# history = "User:ParamBot/ReportHistory" # The default is the rules page's /Instructions subpage. # instructions = "User:ParamBot/Rules/Instructions" # Every edit summary links to the FAQ. diff --git a/docs/documentation.mediawiki b/docs/documentation.mediawiki index 441a227..26931f3 100644 --- a/docs/documentation.mediawiki +++ b/docs/documentation.mediawiki @@ -35,6 +35,9 @@ ! Report | [[User:ParamBot/Report]] |- +! Report history +| [[User:ParamBot/ReportHistory]] +|- ! Emergency stop | [[User:ParamBot/Run]]: change it to no |- @@ -132,6 +135,7 @@ Four times a day, ParamBot: # '''Looks at those categories''', 50 at a time, and skips the empty ones. # '''Fixes each article''' in the categories that have pages. Before each edit, it checks [[User:ParamBot/Run]] again. # '''Writes the report''' to [[User:ParamBot/Report]]. +# '''Adds a line for the run''' to the top of [[User:ParamBot/ReportHistory]]: when it ran, what it did, and how it ended. It stops early if [[User:ParamBot/Run]] stops saying yes, if five articles in a row fail with errors, or as soon as an edit fails in a way that every other edit would too, such as when the bot is blocked. There's no limit on how many edits a run makes, but a run with more than 500 gets a note on the report, so that a rule catching more than it should is noticed. A run stops after 5 hours, so it's always finished before the next run starts, 6 hours later; whatever is left is done then. During a trial approved at [[WP:BRFA|Bots/Requests for approval]], it counts its edits across runs, stops at the number approved, and shows the count on [[User:ParamBot/Report]]. Each trial edit's summary starts with "BRFA trial", linked to the request. @@ -194,6 +198,9 @@ Until then, the report notes that the page has changed, with a link to the chang * merge is only used for lists, never for dates, numbers, coordinates or map names * the report shows no problems for the template, which is easiest to check by approving the new revision under Inactive first +== The report history == +[[User:ParamBot/ReportHistory]] has a line for every run, newest first, except runs that didn't start because [[User:ParamBot/Run]] was off: when it started and ended, in UTC, how long it took, whether it was live, a BRFA trial or reporting only, how many categories had pages, and how many pages it checked, edited and left for review. The last column says how the run ended: finished; finished early, and why; stopped by an error; or aborted because [[User:ParamBot/Run]] was changed, saying when and by whom. It links to the report the run saved. A run aborted by the Run page saves nothing else, but still adds its line here. The bot keeps the newest 200 lines; the text above the table can be edited. + == The report == [[User:ParamBot/Report]] is rewritten on every run. Its "Last run" line says when the run started and ended, in UTC, how long it took, and how many rules pages and rules it read, so the page's history is a log of the runs. Wherever it names a rules page, a template or a diff, it links to it. Its sections: @@ -260,6 +267,8 @@ Until then, the report notes that the page has changed, with a link to the chang |- | [[User:ParamBot/Report]] || What the bot couldn't do, rewritten each run || Must stay editable by the bot |- +| [[User:ParamBot/ReportHistory]] || A line for each run, added by the bot || Must stay editable by the bot +|- | [[User:ParamBot/Run]] || The emergency stop: yes to run, report to update only the report, anything else to stop || '''None''', so anyone can stop the bot |- | [[User talk:ParamBot]] || Questions and problems || — diff --git a/docs/operators.md b/docs/operators.md index 29711a1..6d0c6c1 100644 --- a/docs/operators.md +++ b/docs/operators.md @@ -14,6 +14,7 @@ is your Toolforge shell name. |---|---| | [User:ParamBot/Run](https://en.wikipedia.org/wiki/User:ParamBot/Run) | The off switch: `yes` to run, `report` to update only the report, anything else to stop. | | [User:ParamBot/Report](https://en.wikipedia.org/wiki/User:ParamBot/Report) | What the last run found: pages needing review, pages it skipped, and problems with the rules. | +| [User:ParamBot/ReportHistory](https://en.wikipedia.org/wiki/User:ParamBot/ReportHistory) | A line for every live or report-only run, newest first: when it ran, what it did, and how it ended, including runs aborted by the Run page. | | [User:ParamBot/Rules](https://en.wikipedia.org/wiki/User:ParamBot/Rules) | The approved rules pages, with the revision of each to use. Template-editor protected. | | [Special:Contributions/ParamBot](https://en.wikipedia.org/wiki/Special:Contributions/ParamBot) | Its edits. | | [The BRFA](https://en.wikipedia.org/wiki/Wikipedia:Bots/Requests_for_approval/ParamBot) | The approval request. | @@ -64,7 +65,10 @@ Each run: 5. saves User:ParamBot/Report, every run. Its first line says when the run started and ended and how long it took, and its last line names the commit, such as ``. So the report's history is a - log of the runs. + log of the runs; +6. adds a line to the top of User:ParamBot/ReportHistory, however the run + ended: finished, finished early (and why), stopped by an error, or aborted + because someone changed User:ParamBot/Run (with when, and by whom). A run stops starting new work after 5 hours (the job's `--max-hours 5`), so it can't still be going when the next one starts, 6 hours later. If you change @@ -102,7 +106,8 @@ tail -n 40 ~/parambot.err # the log of the latest run tail -n 3 ~/parambot.out # one line per run: "Last run: ... 3 edits. Rules: ..." ``` -Or look at User:ParamBot/Report and its history. +Or look at User:ParamBot/ReportHistory, a line per run, and +User:ParamBot/Report. ### What would it have edited? diff --git a/parambot/bot.py b/parambot/bot.py index fc0cef2..8f09e5e 100644 --- a/parambot/bot.py +++ b/parambot/bot.py @@ -14,6 +14,7 @@ from . import messages as msg from .botpages import check_bot_pages from .fixer import FixResult, TemplateRules, fix_wikitext +from .history import history_row, updated_page from .options import Options from .prepare import Prepared, prepare from .report import Links, Report @@ -39,6 +40,10 @@ class StopRun(Exception): """The bot mustn't run, or must stop: the message says why.""" +class RunPageOff(StopRun): + """The Run page doesn't say to run: someone has switched the bot off.""" + + # The wiki's error codes for a save that fails in a way every other save in # the run would too: going on would only fail page after page. CANNOT_EDIT_CODES = frozenset({'session-page-restricted', 'blocked', 'autoblocked', @@ -70,6 +75,10 @@ def __init__(self, wiki: Wiki, options: Options) -> None: self._started = _clock() self._trial_made: int | None = None # edits made in a BRFA trial, if there is one self._trial_log: list[str] = [] # this run's trial edits, for the trial's log + self._history = False # whether the run gets a line on the history page + self._switched_off: RunPageOff | None = None # the Run page stopped the run + self._ended_early: str | None = None # why the run stopped before the last page + self._report_revision: int | None = None # the revision of the report it saved # -- a run ------------------------------------------------------------- @@ -78,31 +87,53 @@ def run(self) -> Report: self.report.started = _now() self._start_trial() self._preflight() + # From here on, a run that saves the report adds a line to the + # history page, however it ends. + self._history = self.options.saves_report try: self._run() except StopRun as stop: - # Someone switched the bot off, or it can't start: make no more - # wiki edits, not even the report or the trial's log. + # Someone switched the bot off, or it can't go on: make no more + # wiki edits, not even the report or the trial's log. Only the + # line on the history page, in the bot's own userspace, which + # says why the run stopped. self.report.errors.append(msg.stopped_early(stop)) if self._trial_log: log.warning('%s', msg.plain(msg.trial_log_not_saved( self.options.trial_log_page, len(self._trial_log), self._trial_log_file))) self._write_report(local_only=True) + if isinstance(stop, RunPageOff): + self._write_history('aborted by the Run page', self._switched_off_result(stop)) + else: + self._write_history('stopped', 'stopped: ' + self.report.links.wikitext(str(stop))) raise except CannotEdit as stop: log.error('%s', msg.plain(str(stop))) self.report.errors.append(str(stop)) self._write_trial_log() self._write_report() + self._write_history('stopped by an error', + 'error: ' + self.report.links.wikitext(str(stop))) raise except Exception as error: log.exception('Run failed') self.report.errors.append(msg.run_failed(error)) self._write_trial_log() self._write_report() + self._write_history('stopped by an error', + 'error: ' + self.report.links.wikitext(msg.run_failed(error))) raise self._write_trial_log() self._write_report() + if self._switched_off is not None: + # Reporting only, and switched off before the report was saved. + self._write_history('aborted by the Run page', + self._switched_off_result(self._switched_off)) + elif self._ended_early: + self._write_history('finished early', 'finished early: ' + + self.report.links.wikitext(self._ended_early)) + else: + self._write_history('finished', 'finished') return self.report def check_rules(self) -> Report: @@ -202,8 +233,8 @@ def _check_trial_log_page(self) -> None: title = normalize_title(options.trial_log_page) others = {normalize_title(t) for t in ( options.user_page, options.rules_page, options.run_page, options.report_page, - options.instructions_page, options.faq_page, options.header_page, - options.link_rule_page)} + options.history_page, options.instructions_page, options.faq_page, + options.header_page, options.link_rule_page)} if (not self._in_own_userspace(title) or title in others or title.startswith(normalize_title(options.rules_page) + '/')): raise StopRun(msg.trial_log_elsewhere(options.trial_log_page, options.user_page)) @@ -219,6 +250,7 @@ def _preflight(self) -> None: self._check_trial_log_page() if options.saves_report: self._check_report_page() + self._check_history_page() self._check_account() self._check_run_page() problems = self._check_pages() @@ -279,6 +311,20 @@ def _check_report_page(self) -> None: if options.report_only and not self._in_own_userspace(report): raise StopRun(msg.report_only_elsewhere(options.report_page, options.user_page)) + def _check_history_page(self) -> None: + """The history page is saved on every run that saves the report, even + a report-only one, so it must be the bot's own page, and not one of its + others.""" + options = self.options + title = normalize_title(options.history_page) + others = {normalize_title(t) for t in ( + options.user_page, options.rules_page, options.run_page, options.report_page, + options.instructions_page, options.faq_page, options.header_page, + options.link_rule_page, options.trial_log_page)} + if (not self._in_own_userspace(title) or title in others + or title.startswith(normalize_title(options.rules_page) + '/')): + raise StopRun(msg.history_elsewhere(options.history_page, options.user_page)) + def _in_own_userspace(self, title: str) -> bool: return normalize_title(title).startswith(normalize_title(self.options.user_page) + '/') @@ -299,18 +345,19 @@ def _check_run_page(self, before: str | None = None) -> None: text = strip_comments(page.text if page.exists() else '').strip().lower() allowed = RUN_VALUES | REPORT_VALUES if self.options.report_only else RUN_VALUES if text not in allowed: - raise StopRun(msg.run_page_off(self.options.run_page, before, + raise RunPageOff(msg.run_page_off(self.options.run_page, before, self.options.report_only)) def _check_may_save(self, title: str) -> None: """The last check before anything is saved. Reporting only, nothing - but the report page in the bot's own userspace, whatever else the run - was told.""" + but the report and history pages in the bot's own userspace, whatever + else the run was told.""" options = self.options + allowed = {normalize_title(options.report_page), normalize_title(options.history_page)} if options.report_only and ( - normalize_title(title) != normalize_title(options.report_page) - or not self._in_own_userspace(title)): - raise RuntimeError(msg.report_only_refused_save(title, options.report_page)) + normalize_title(title) not in allowed or not self._in_own_userspace(title)): + raise RuntimeError(msg.report_only_refused_save(title, options.report_page, + options.history_page)) def _check_pages(self) -> list[str]: """Problems with the bot's own pages; notes go straight on the report.""" @@ -397,13 +444,13 @@ def _process_all(self, candidates: dict[str, Candidate], pages: Iterable[WikiPag failures = 0 for page in pages: if self._trial_full(): - self.report.notes.append(msg.stopped_at_trial_limit(self.options.trial_edits)) + self._end_early(msg.stopped_at_trial_limit(self.options.trial_edits)) break if self.options.max_edits and self.report.edits >= self.options.max_edits: - self.report.notes.append(msg.stopped_at_max_edits(self.options.max_edits)) + self._end_early(msg.stopped_at_max_edits(self.options.max_edits)) break if self._out_of_time(): - self.report.notes.append(msg.stopped_at_max_hours(self.options.max_hours or 0)) + self._end_early(msg.stopped_at_max_hours(self.options.max_hours or 0)) break candidate = candidates.get(page.title()) if candidate is None: @@ -422,6 +469,10 @@ def _process_all(self, candidates: dict[str, Candidate], pages: Iterable[WikiPag else: failures = 0 + def _end_early(self, why: str) -> None: + self.report.notes.append(why) + self._ended_early = why + def _process(self, page: WikiPage, targets: list[TemplateRules]) -> None: title = page.title() reason = self._not_checkable(page) @@ -554,6 +605,8 @@ def _write_report(self, local_only: bool = False) -> None: saved = True except StopRun as stop: log.warning('Not saving %s: %s', options.report_page, msg.plain(str(stop))) + if isinstance(stop, RunPageOff): + self._switched_off = stop except Exception: log.exception('Could not save %s; writing the report locally instead', options.report_page) @@ -583,6 +636,54 @@ def _save_report_page(self, text: str) -> None: page.save(summary=msg.report_summary(self.report.edits, len(self.report.issues), self.options.report_only), minor=True, bot=self.wiki.has_right('bot'), quiet=True, nocreate=True) + self._report_revision = page.latest_revision_id + + # -- the report history ------------------------------------------------ + + def _write_history(self, status: str, result: str) -> None: + """Add a line for this run to the top of the history page, saying how + it ended: result, as wikitext, and status, in a few words for the + edit summary. Only for runs that save the report, once they've + started. Never raises: the history is only a record.""" + if not self._history: + return + options, report = self.options, self.report + if self._report_revision: + result += f' ([[Special:Permalink/{self._report_revision}|report]])' + mode = 'reporting only' if options.report_only else ( + 'live (BRFA trial)' if options.in_trial else 'live') + assert report.started is not None and report.ended is not None + row = history_row( + started=report.started, ended=report.ended, mode=mode, + categories=f'{report.categories_populated} of {report.categories_polled}', + checked=report.pages_checked, edits=report.edits, + needing_review=len({title for title, _ in report.issues}), result=result) + intro = msg.history_intro(options.bot_user, options.report_page, options.run_page) + try: + self._check_may_save(options.history_page) + page = self.wiki.page(options.history_page) + old = page.text if page.exists() else None + page.text = updated_page(old, row, header=options.header_page, intro=intro) + # Created if need be: it's in the bot's own userspace (see + # _check_history_page). + page.save(summary=msg.history_summary(status), minor=True, + bot=self.wiki.has_right('bot'), quiet=True) + except Exception: + log.exception('Could not add this run to %s', options.history_page) + else: + log.info('Added this run to %s', options.history_page) + + def _switched_off_result(self, stop: RunPageOff) -> str: + """How a run someone stopped with the Run page ended: when the page + was changed, and by whom, from its newest revision.""" + try: + change = self.wiki.last_change(self.options.run_page) + except Exception as error: + log.warning('Could not read when %s was changed: %s', self.options.run_page, error) + change = None + when, user = change or (None, None) + return (msg.run_page_changed(self.options.run_page, when, user) + ': ' + + self.report.links.wikitext(str(stop))) def trial_log_line(revid: int | None, title: str, when: datetime) -> str: diff --git a/parambot/cli.py b/parambot/cli.py index abc7e52..24c7206 100644 --- a/parambot/cli.py +++ b/parambot/cli.py @@ -127,6 +127,9 @@ def _build_parser() -> argparse.ArgumentParser: run.add_argument('--report-page', help='default: [pages] report, else User:/Report') run.add_argument('--run-page', help='default: [pages] run, else User:/Run') + run.add_argument('--history-page', + help='where live and report-only runs add a line for each run (default: ' + '[pages] history, else User:/ReportHistory)') run.add_argument('--out-dir', help="where dry runs (and live runs that can't save the report) write " 'files (default: [output] dir, else out)') @@ -175,7 +178,7 @@ def _options(args: argparse.Namespace, settings: Settings | None = None, templates=tuple(args.template), pages=tuple(args.page), any_namespace=args.any_namespace) for name in ('max_edits', 'max_hours', 'cooldown_days', 'report_page', 'run_page', - 'out_dir'): + 'history_page', 'out_dir'): if getattr(args, name) is not None: values[name] = getattr(args, name) return Options(**values) diff --git a/parambot/history.py b/parambot/history.py new file mode 100644 index 0000000..cfeaf8b --- /dev/null +++ b/parambot/history.py @@ -0,0 +1,73 @@ +"""The report history page, User:ParamBot/ReportHistory: a row for each run. + +The report page is replaced on every run, so it only says what the latest +one found. The history page keeps a line for every live and report-only +run, newest first, saying when it ran, what it did and how it ended, except +runs that didn't start because User:ParamBot/Run was off: + + {{User:ParamBot/Header}} + A line for every run of ParamBot, newest first, except runs that didn't + start because User:ParamBot/Run was off. ... + + {| class="wikitable" + ! Started (UTC) !! Ended (UTC) !! Duration !! Mode !! Categories !! Checked !! Edits + !! Needs review !! Result + + |- + | 2026-10-27 04:00 || 2026-10-27 04:12 || 12 minutes || live || 3 of 41 || 57 || 52 || 2 + || finished ([[Special:Permalink/123|report]]) + |} + +(the row is one line on the page). A run that someone stopped with the Run +page says when the page was changed, and by whom. Everything above the +marker is kept as it is, so the introduction can be edited by hand; the +oldest rows go once there are MAX_ROWS. +""" + +from datetime import datetime + +from . import messages as msg + +__all__ = ['MARKER', 'MAX_ROWS', 'updated_page', 'history_row'] + +MARKER = '' +MAX_ROWS = 200 # 50 days of runs every 6 hours +_TABLE_END = '|}' +_HEADINGS = ('! Started (UTC) !! Ended (UTC) !! Duration !! Mode !! Categories !! Checked ' + '!! Edits !! Needs review !! Result') + + +def updated_page(old: str | None, row: str, *, header: str, intro: str) -> str: + """The history page's text with row at the top of its table. A page + without the table (new, a placeholder, or edited by hand) is started + afresh.""" + if old is None or MARKER not in old or _TABLE_END not in old.split(MARKER, 1)[1]: + top = f'{{{{{header}}}}}\n' if header else '' + return (f'{top}{intro}\n\n{{| class="wikitable"\n{_HEADINGS}\n' + f'{MARKER}\n{row.rstrip(chr(10))}\n{_TABLE_END}\n') + head, rest = old.split(MARKER, 1) + body, _, tail = rest.partition('\n' + _TABLE_END) + rows = [row] + _rows(body)[:MAX_ROWS - 1] + return head + MARKER + '\n' + ''.join(r.rstrip('\n') + '\n' for r in rows) + _TABLE_END + tail + + +def _rows(body: str) -> list[str]: + """The table's rows, each starting with its |- line.""" + rows: list[str] = [] + for line in body.strip('\n').split('\n'): + if line.startswith('|-') or not rows: + rows.append('') + rows[-1] += line + '\n' + return [row for row in rows if row.strip()] + + +def history_row(*, started: datetime, ended: datetime, mode: str, categories: str, + checked: int, edits: int, needing_review: int, result: str) -> str: + """A run's row. result is wikitext, saying how the run ended.""" + cells = [_when(started), _when(ended), msg.duration((ended - started).total_seconds()), + mode, categories, str(checked), str(edits), str(needing_review), result] + return '|-\n| ' + ' || '.join(cells) + '\n' + + +def _when(moment: datetime) -> str: + return f'{moment:%Y-%m-%d %H:%M}' diff --git a/parambot/messages.py b/parambot/messages.py index 802d5ba..8855aa3 100644 --- a/parambot/messages.py +++ b/parambot/messages.py @@ -688,8 +688,9 @@ def report_only_elsewhere(report_page: str, user_page: str) -> str: f'to {report_page}. Make the report page a subpage of {user_page}.') -def report_only_refused_save(title: str, report_page: str) -> str: - return f'Reporting only: refused to save {title}. The only page it may edit is {report_page}.' +def report_only_refused_save(title: str, report_page: str, history_page: str) -> str: + return (f'Reporting only: refused to save {title}. The only pages it may edit are ' + f'{report_page} and {history_page}.') def report_page_taken(title: str, what: str) -> str: @@ -708,3 +709,31 @@ def stopped_early(reason: object) -> str: def run_failed(error: Exception) -> str: return (f'The run stopped early because of an error: {type(error).__name__}: ' f'{quoted(error)}') + + +# -- the report history ---------------------------------------------------- + +def history_intro(bot_user: str, report_page: str, run_page: str) -> str: + return (f'A line for every run of [[User:{bot_user}|{bot_user}]], newest first, except runs ' + f"that didn't start because [[{run_page}]] was off: when it ran, what it did and how " + 'it ended. The bot adds a line at the end of each live or report-only run, however ' + f'it ends. What the latest run found is on [[{report_page}]]; a run stopped by ' + f'changing [[{run_page}]] says so here.') + + +def history_summary(result: str) -> str: + return f'Adding a line for this run: {result}' + + +def history_elsewhere(page: str, user_page: str) -> str: + return (f'{page}, the report history, must be a page of its own in {user_page}\'s ' + 'userspace, since the bot edits it on every run. Change ' + f'{code("[pages] history")} in the settings.') + + +def run_page_changed(run_page: str, when: datetime | None, user: str | None) -> str: + """How a run that someone stopped with the Run page ended, for the + report history: wikitext.""" + at = f' at {when:%Y-%m-%d %H:%M}' if when else '' + by = f' by [[User:{user}|{user}]]' if user else '' + return f"aborted: [[{run_page}]] was changed{at}{by}, so the run stopped" diff --git a/parambot/options.py b/parambot/options.py index 33e813c..cdb33da 100644 --- a/parambot/options.py +++ b/parambot/options.py @@ -17,6 +17,7 @@ class Options: rules_page: str = '' report_page: str = '' run_page: str = '' + history_page: str = '' # a line for each run that saves the report instructions_page: str = '' # default: /Instructions faq_page: str = '' # every edit summary links to it header_page: str = '' # the links across the top of the bot's pages and the report @@ -62,6 +63,7 @@ def __post_init__(self) -> None: self.rules_page = self.rules_page or f'{base}/Rules' self.report_page = self.report_page or f'{base}/Report' self.run_page = self.run_page or f'{base}/Run' + self.history_page = self.history_page or f'{base}/ReportHistory' self.instructions_page = self.instructions_page or f'{self.rules_page}/Instructions' self.faq_page = self.faq_page or f'{base}/FAQ' self.header_page = self.header_page or f'{base}/Header' @@ -99,6 +101,7 @@ def other_pages(self) -> dict[str, str]: self.user_page: "the bot's user page", self.rules_page: 'the rules page', self.run_page: 'the Run page', + self.history_page: 'the report history', self.instructions_page: 'the instructions for rule writers', self.faq_page: 'the FAQ', self.header_page: 'the header', diff --git a/parambot/settings.py b/parambot/settings.py index 3dfb342..8a4314e 100644 --- a/parambot/settings.py +++ b/parambot/settings.py @@ -47,6 +47,7 @@ class _Setting: 'rules': _Setting(str, 'rules_page'), 'report': _Setting(str, 'report_page'), 'run': _Setting(str, 'run_page'), + 'history': _Setting(str, 'history_page'), 'instructions': _Setting(str, 'instructions_page'), 'faq': _Setting(str, 'faq_page'), 'header': _Setting(str, 'header_page'), diff --git a/parambot/wiki.py b/parambot/wiki.py index 55b324d..1072f14 100644 --- a/parambot/wiki.py +++ b/parambot/wiki.py @@ -127,6 +127,15 @@ def revisions(self, revids: Iterable[int]) -> dict[int, Revision]: page['lastrevid']) return found + def last_change(self, title: str) -> tuple[datetime, str] | None: + """When the page was last edited, and by whom; None if it doesn't + exist. Fetched afresh.""" + page = pywikibot.Page(self.site, title) + if not page.exists(): + return None + revision = page.latest_revision + return revision.timestamp, str(revision.user) + def subpages(self, title: str) -> list[str]: """The titles of the pages under title/, without redirects: a request for each 500 of them (5,000 with the bot right).""" diff --git a/tests/fakes.py b/tests/fakes.py index 6e356e0..977bf7e 100644 --- a/tests/fakes.py +++ b/tests/fakes.py @@ -83,8 +83,10 @@ class FakePage: def __init__(self, title, text='', *, exists=True, redirect_to=None, model='wikitext', templates=(), protection=None, editable=True, may_edit=True, revisions=(), redirects=(), broken=False, save_error=None, on_save=None, - revid=None, history=None, deleted_before_save=False): + revid=None, history=None, deleted_before_save=False, wiki=None): self._title, self.text = title, text + # The wiki a missing page joins if it's saved: created, as on a real wiki. + self._wiki = wiki # Deleted after the bot loaded it: a save recreates it unless the # wiki is told not to create pages. self._deleted_before_save = deleted_before_save @@ -139,6 +141,9 @@ def save(self, *, summary, minor, bot, quiet, nocreate=False): raise self._save_error if nocreate and (self._deleted_before_save or not self._exists): raise pwb_exc.NoCreateError(0) # MediaWiki's "missingtitle" + self._exists = True # created, if it wasn't there + if self._wiki is not None: + self._wiki.pages.setdefault(self._title, self) self.text = self.text.rstrip() # as MediaWiki saves it self.saved.append(self.text) self.summaries.append(summary) @@ -191,7 +196,7 @@ def has_right(self, right): def page(self, title, ns=0): if ns == 10 and not title.startswith('Template:'): title = 'Template:' + title - return self.pages.get(title) or FakePage(title, exists=False) + return self.pages.get(title) or FakePage(title, exists=False, wiki=self) def load(self, pages, *, templates=False): pages = list(pages) @@ -215,6 +220,13 @@ def revisions(self, revids): page.latest_revision_id) return found + def last_change(self, title): + page = self.pages.get(title) + if page is None or not page._exists or not page._revisions: + return None + revision = max(page._revisions, key=lambda r: r.timestamp) + return revision.timestamp, revision.user + def subpages(self, title): self.requests['subpages'] += 1 return sorted(name for name, page in self.pages.items() diff --git a/tests/test_bot.py b/tests/test_bot.py index 69e92e4..f5b6e59 100644 --- a/tests/test_bot.py +++ b/tests/test_bot.py @@ -13,6 +13,7 @@ OFFICEHOLDER_RULES, TEMPLATE_EDITOR, FakePage, + FakeRevision, FakeWiki, edited_by, link_rule, @@ -28,6 +29,7 @@ SUMMARY_LIMIT, CannotEdit, ParamBot, + RunPageOff, StopRun, edit_summary, read_trial_count, @@ -949,6 +951,213 @@ def test_dry_run_carries_on_and_reports_setup_problems(tmp_path): assert 'is not protected' in text +# -- the report history ---------------------------------------------------- + +def history_rows(wiki, opts): + """The rows on the history page, newest first, each as one line.""" + text = wiki.page(opts.history_page).text + return [row.replace('\n', ' ').strip() for row in text.split('|-\n')[1:]] + + +def test_a_live_run_adds_a_line_to_the_history_page(tmp_path, monkeypatch): + times = iter([datetime(2026, 10, 27, 4, 0, tzinfo=UTC), + datetime(2026, 10, 27, 4, 12, tzinfo=UTC)]) + monkeypatch.setattr('parambot.bot._now', lambda: next(times)) + opts = options(live=True, out_dir=str(tmp_path)) + wiki = wiki_for(opts).populate(OFFICEHOLDER_CATEGORY, article('Jane Example')) + ParamBot(wiki, opts).run() + history = wiki.page(opts.history_page) + assert opts.history_page == f'User:{BOT}/ReportHistory' + assert history.exists() # created by the first run + assert history.summaries == ['Adding a line for this run: finished'] + assert history.text.startswith( + f'{{{{User:{BOT}/Header}}}}\nA line for every run of [[User:{BOT}|{BOT}]], newest ' + f"first, except runs that didn't start because [[User:{BOT}/Run]] was off: ") + revision = wiki.page(opts.report_page).latest_revision_id + assert history_rows(wiki, opts) == [ + '| 2026-10-27 04:00 || 2026-10-27 04:12 || 12 minutes || live || 1 of 1 || 1 || 1 || 0 ' + f'|| finished ([[Special:Permalink/{revision}|report]]) |}}'] + + +def test_each_run_adds_its_own_line_newest_first(tmp_path): + opts = options(live=True, out_dir=str(tmp_path)) + wiki = wiki_for(opts) + ParamBot(wiki, opts).run() + first = wiki.page(opts.report_page).latest_revision_id + ParamBot(wiki, opts).run() + second = wiki.page(opts.report_page).latest_revision_id + rows = history_rows(wiki, opts) + assert len(rows) == 2 + assert f'Permalink/{second}|' in rows[0] and f'Permalink/{first}|' in rows[1] + + +def test_a_dry_run_adds_no_line(tmp_path): + opts = options(out_dir=str(tmp_path)) + wiki = wiki_for(opts) + ParamBot(wiki, opts).run() + assert wiki.page(opts.history_page).saved == [] + + +def test_reporting_only_adds_a_line_too(tmp_path): + opts = options(report_only=True, out_dir=str(tmp_path)) + wiki = wiki_for(opts).populate(OFFICEHOLDER_CATEGORY, article('Jane Example')) + wiki.rights = set() + ParamBot(wiki, opts).run() + [row] = history_rows(wiki, opts) + assert '|| reporting only || 1 of 1 || 1 || 1 || 0 || finished' in row + + +def test_a_trial_says_so_on_the_history_page(tmp_path): + opts = trial(tmp_path) + wiki = wiki_for(opts).populate(OFFICEHOLDER_CATEGORY, article('Jane Example')) + ParamBot(wiki, opts).run() + [row] = history_rows(wiki, opts) + assert '|| live (BRFA trial) ||' in row + + +def test_switched_off_mid_run_the_history_says_who_aborted_it(tmp_path): + opts = options(live=True, out_dir=str(tmp_path)) + wiki = wiki_for(opts) + changed = datetime(2026, 10, 27, 4, 7, tzinfo=UTC) + + def switch_off(): + run_page = wiki.page(opts.run_page) + run_page.text = 'no' + run_page._revisions.append(FakeRevision('Some editor', changed)) + first, second = article('First', on_save=switch_off), article('Second') + wiki.populate(OFFICEHOLDER_CATEGORY, first, second) + with pytest.raises(RunPageOff): + ParamBot(wiki, opts).run() + assert wiki.page(opts.report_page).saved == [] # still not the report + [row] = history_rows(wiki, opts) + # Second was checked, but not edited. + assert ('|| live || 1 of 1 || 2 || 1 || 0 || aborted: [[User:ExampleBot/Run]] was changed ' + 'at 2026-10-27 04:07 by [[User:Some editor|Some editor]], so the run stopped: ' + 'User:ExampleBot/Run no longer says "yes"; stopped before editing Second') in row + assert 'Permalink' not in row # no report was saved + assert wiki.page(opts.history_page).summaries == [ + 'Adding a line for this run: aborted by the Run page'] + + +def test_the_history_explains_an_abort_even_without_the_run_pages_history(tmp_path): + opts = options(live=True, out_dir=str(tmp_path)) + wiki = wiki_for(opts) + + def switch_off(): + wiki.page(opts.run_page).text = 'no' + wiki.populate(OFFICEHOLDER_CATEGORY, article('First', on_save=switch_off), article('Second')) + wiki.last_change = lambda title: (_ for _ in ()).throw(ConnectionError('API down')) + with pytest.raises(RunPageOff): + ParamBot(wiki, opts).run() + [row] = history_rows(wiki, opts) + assert '|| aborted: [[User:ExampleBot/Run]] was changed, so the run stopped: ' in row + + +def test_reporting_only_switched_off_mid_run_is_explained_in_the_history(tmp_path): + opts = options(report_only=True, out_dir=str(tmp_path)) + wiki = wiki_for(opts).populate(OFFICEHOLDER_CATEGORY, article('Jane Example')) + members = wiki.category_members + + def switched_off_while_polling(*args): + wiki.page(opts.run_page).text = 'no' + return members(*args) + wiki.category_members = switched_off_while_polling + ParamBot(wiki, opts).run() + assert wiki.page(opts.report_page).saved == [] + [row] = history_rows(wiki, opts) + assert ('|| reporting only || 1 of 1 || 1 || 1 || 0 || aborted: [[User:ExampleBot/Run]] ' + 'was changed, so the run stopped: User:ExampleBot/Run no longer says') in row + + +def test_switched_off_before_the_run_adds_no_line(tmp_path): + # Every scheduled run while the bot is off would add one otherwise. + opts = options(live=True, out_dir=str(tmp_path)) + wiki = wiki_for(opts, **{opts.run_page: FakePage(opts.run_page, 'no')}) + with pytest.raises(StopRun): + ParamBot(wiki, opts).run() + assert wiki.page(opts.history_page).saved == [] + + +def test_a_run_that_ends_early_says_why_in_the_history(tmp_path): + opts = options(live=True, max_edits=1, out_dir=str(tmp_path)) + wiki = wiki_for(opts).populate(OFFICEHOLDER_CATEGORY, article('First'), article('Second')) + ParamBot(wiki, opts).run() + [row] = history_rows(wiki, opts) + assert '|| finished early: Stopped after 1 edit (max_edits). ([[Special:Permalink/' in row + assert wiki.page(opts.history_page).summaries == [ + 'Adding a line for this run: finished early'] + + +def test_a_crash_is_on_the_history_page(tmp_path): + opts = options(live=True, out_dir=str(tmp_path)) + wiki = wiki_for(opts) + wiki.fail_polling = ConnectionError('API down') + with pytest.raises(ConnectionError): + ParamBot(wiki, opts).run() + [row] = history_rows(wiki, opts) + assert '|| error: The run stopped early because of an error: ConnectionError: ' in row + assert 'API down' in row + + +def test_a_save_error_every_edit_would_hit_is_on_the_history_page(tmp_path): + opts = options(live=True, out_dir=str(tmp_path)) + blocked = pwb_exc.OtherPageSaveError(0, pwb_exc.APIError('blocked', 'Blocked')) + wiki = wiki_for(opts).populate(OFFICEHOLDER_CATEGORY, article('Jane', save_error=blocked)) + with pytest.raises(CannotEdit): + ParamBot(wiki, opts).run() + [row] = history_rows(wiki, opts) + assert '|| error: Stopped before editing any more articles' in row + + +def test_a_stop_other_than_the_run_page_is_on_the_history_page(tmp_path, monkeypatch): + opts = trial(tmp_path) + wiki = wiki_for(opts).populate(OFFICEHOLDER_CATEGORY, article('Jane Example')) + + def cannot_save(path, made): + raise OSError('disk full') + monkeypatch.setattr('parambot.bot.write_trial_count', cannot_save) + with pytest.raises(StopRun): + ParamBot(wiki, opts).run() + [row] = history_rows(wiki, opts) + assert '|| stopped: ' in row + + +def test_a_history_page_that_cannot_be_saved_does_not_stop_the_run(tmp_path, caplog): + opts = options(live=True, out_dir=str(tmp_path)) + unsaveable = FakePage(opts.history_page, '', save_error=ConnectionError('wiki unreachable')) + wiki = wiki_for(opts, **{opts.history_page: unsaveable}) + wiki.populate(OFFICEHOLDER_CATEGORY, article('Jane Example')) + assert ParamBot(wiki, opts).run().edits == 1 + assert wiki.page(opts.report_page).saved + assert f'Could not add this run to {opts.history_page}' in caplog.messages + + +@pytest.mark.parametrize('history_page', [ + 'Wikipedia:Sandbox', f'User:{BOT}Two/ReportHistory', f'User:{BOT}/Run', + f'User:{BOT}/Rules/Infobox officeholder', f'User:{BOT}/BRFA Log']) +def test_the_history_page_must_be_the_bots_own(tmp_path, history_page): + opts = options(live=True, history_page=history_page, out_dir=str(tmp_path)) + wiki = wiki_for(opts) + with pytest.raises(StopRun, match='the report history, must be a page of its own'): + ParamBot(wiki, opts).run() + assert not wiki.logged_in + + +def test_the_report_never_replaces_the_history_page(tmp_path): + opts = options(live=True, report_page=f'User:{BOT}/ReportHistory', out_dir=str(tmp_path)) + with pytest.raises(StopRun, match='which is the report history'): + ParamBot(wiki_for(opts), opts).run() + + +def test_reporting_only_may_save_the_history_page_too(tmp_path): + opts = options(report_only=True, out_dir=str(tmp_path)) + bot = ParamBot(wiki_for(opts), opts) + bot._check_may_save(opts.history_page) + with pytest.raises(RuntimeError, match=f'may edit are {opts.report_page} and ' + f'{opts.history_page}'): + bot._check_may_save('Wikipedia:Sandbox') + + # -- failures -------------------------------------------------------------- def test_crash_still_writes_the_report(tmp_path): diff --git a/tests/test_history.py b/tests/test_history.py new file mode 100644 index 0000000..eccecac --- /dev/null +++ b/tests/test_history.py @@ -0,0 +1,52 @@ +"""The report history page: a line for each run.""" + +from datetime import UTC, datetime, timedelta + +import pytest + +from parambot.history import MARKER, MAX_ROWS, history_row, updated_page + +STARTED = datetime(2026, 10, 27, 4, 0, tzinfo=UTC) + + +def row(result='finished', minutes=12): + return history_row(started=STARTED, ended=STARTED + timedelta(minutes=minutes), mode='live', + categories='3 of 41', checked=57, edits=52, needing_review=2, + result=result) + + +def test_a_row_is_one_line_of_the_table(): + assert row() == ('|-\n| 2026-10-27 04:00 || 2026-10-27 04:12 || 12 minutes || live || 3 of 41 ' + '|| 57 || 52 || 2 || finished\n') + + +@pytest.mark.parametrize('old', [None, '', 'Placeholder.', f'{MARKER}\nno table end']) +def test_a_page_without_the_table_is_started_afresh(old): + text = updated_page(old, row(), header='User:ExampleBot/Header', intro='Every run.') + assert text.startswith('{{User:ExampleBot/Header}}\nEvery run.\n\n{| class="wikitable"\n') + assert text.endswith(f'{MARKER}\n{row()}|}}\n') + + +def test_new_runs_go_at_the_top_and_the_rest_is_kept(): + first = updated_page(None, row('first'), header='', intro='Intro.') + edited = first.replace('Intro.', 'Intro, edited by hand.') + 'A note below.\n' + text = updated_page(edited, row('second'), header='', intro='Intro.') + assert 'Intro, edited by hand.' in text + assert text.index('second') < text.index('first') + assert text.endswith('|}\nA note below.\n') + + +def test_the_oldest_rows_go(): + text = None + for n in range(MAX_ROWS + 5): + text = updated_page(text, row(f'run {n}'), header='', intro='') + assert text.count('|-\n') == MAX_ROWS + assert f'run {MAX_ROWS + 4}' in text + assert 'run 4\n' not in text and 'run 5\n' in text + + +@pytest.mark.parametrize('minutes, shown', [(14, '14 minutes'), (0.5, 'under a minute'), + (125, '2 hours 5 minutes')]) +def test_a_row_says_how_long_the_run_took_as_the_report_does(minutes, shown): + # The same words as the report's "taking 14 minutes" (messages.duration). + assert f' || {shown} || live || ' in row(minutes=minutes) diff --git a/tests/test_wiki.py b/tests/test_wiki.py index d2c95c9..5b6b7ad 100644 --- a/tests/test_wiki.py +++ b/tests/test_wiki.py @@ -304,3 +304,23 @@ def members(self, *, namespaces): monkeypatch.setattr('parambot.wiki.pywikibot.Category', Category) assert list(Wiki(Pages()).category_members('Category:C', (0,))) == ['Article'] assert asked == ['Category:C', [0]] + + +class Revisions: + """Enough of a Pywikibot page for last_change.""" + + def __init__(self, exists): + self._exists = exists + self.latest_revision = type('Revision', (), { + 'timestamp': datetime(2026, 10, 27, 4, 7, tzinfo=UTC), 'user': 'Some editor'})() + + def exists(self): + return self._exists + + +def test_last_change_says_when_and_by_whom(monkeypatch): + monkeypatch.setattr('parambot.wiki.pywikibot.Page', lambda site, title: Revisions(True)) + assert Wiki(Site()).last_change('User:ParamBot/Run') == ( + datetime(2026, 10, 27, 4, 7, tzinfo=UTC), 'Some editor') + monkeypatch.setattr('parambot.wiki.pywikibot.Page', lambda site, title: Revisions(False)) + assert Wiki(Site()).last_change('User:ParamBot/Run') is None