Skip to content

Commit 677e3fd

Browse files
methanemethane
committed
update
1 parent fb0919c commit 677e3fd

4 files changed

Lines changed: 241 additions & 15 deletions

File tree

‎.github/workflows/update.yml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -107,10 +107,8 @@ jobs:
107107
pip install pomerge powrap
108108
109109
- name: Merge overwriting on stable release branch
110-
shell: bash
111110
run: |
112-
shopt -s globstar nullglob
113-
pomerge --from "${{ env.CPYTHON_BRANCH }}"/**/*.po --to "${{ matrix.branch }}"/**/*.po
111+
python "${{ env.CPYTHON_BRANCH }}/scripts/translations.py" merge "${{ env.CPYTHON_BRANCH }}" "${{ matrix.branch }}"
114112
115113
- name: Wrap catalog message files
116114
run: |

‎scripts/translations.py‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
"""Sync and merge the PO catalogs listed in a generated Transifex config."""
2+
3+
import argparse
4+
import configparser
5+
import os
6+
from pathlib import Path, PurePosixPath
7+
import shutil
8+
import subprocess
9+
10+
11+
def active_catalogs(config_path):
12+
config = configparser.ConfigParser(interpolation=None)
13+
with config_path.open(encoding="utf-8") as stream:
14+
config.read_file(stream)
15+
catalogs = set()
16+
for section in config.sections():
17+
if section == "main":
18+
continue
19+
source = PurePosixPath(config[section]["source_file"])
20+
relative = source.relative_to("pot")
21+
if ".." in relative.parts or relative.suffix != ".pot":
22+
raise ValueError(f"Invalid catalog path: {source}")
23+
catalogs.add(Path(relative.with_suffix(".po")))
24+
if not catalogs:
25+
raise ValueError(f"No catalogs listed in {config_path}")
26+
return sorted(catalogs)
27+
28+
29+
def tracked_catalogs(root):
30+
result = subprocess.run(
31+
["git", "-C", str(root), "ls-files", "-z", "--", "*.po"],
32+
check=True,
33+
stdout=subprocess.PIPE,
34+
)
35+
return [Path(os.fsdecode(name)) for name in result.stdout.split(b"\0") if name]
36+
37+
38+
def require_catalogs(root, catalogs):
39+
missing = [str(path) for path in catalogs if not (root / path).is_file()]
40+
if missing:
41+
raise FileNotFoundError(f"Missing PO catalogs in {root}: {', '.join(missing)}")
42+
43+
44+
def sync(source, destination, language):
45+
config = source / ".tx/config"
46+
catalogs = active_catalogs(config)
47+
downloaded = source / language / "LC_MESSAGES"
48+
require_catalogs(downloaded, catalogs)
49+
obsolete = set(tracked_catalogs(destination)) - set(catalogs)
50+
51+
for path in catalogs:
52+
target = destination / path
53+
target.parent.mkdir(parents=True, exist_ok=True)
54+
shutil.copyfile(downloaded / path, target)
55+
for path in sorted(obsolete):
56+
(destination / path).unlink(missing_ok=True)
57+
print(f"Removed obsolete catalog: {path}")
58+
(destination / ".tx").mkdir(exist_ok=True)
59+
shutil.copyfile(config, destination / ".tx/config")
60+
61+
62+
def merge(source, destination):
63+
# Ignore stale PO files even if a source checkout still contains them.
64+
catalogs = active_catalogs(source / ".tx/config")
65+
targets = tracked_catalogs(destination)
66+
if not targets:
67+
raise ValueError(f"No target catalogs in {destination}")
68+
require_catalogs(source, catalogs)
69+
require_catalogs(destination, targets)
70+
subprocess.run(
71+
[
72+
"pomerge",
73+
"--from",
74+
*(str(source / path) for path in catalogs),
75+
"--to",
76+
*(str(destination / path) for path in targets),
77+
],
78+
check=True,
79+
)
80+
81+
82+
def main():
83+
parser = argparse.ArgumentParser(description=__doc__)
84+
commands = parser.add_subparsers(dest="command", required=True)
85+
sync_parser = commands.add_parser("sync")
86+
sync_parser.add_argument("source", type=Path)
87+
sync_parser.add_argument("destination", type=Path)
88+
sync_parser.add_argument("--language", default="ja")
89+
merge_parser = commands.add_parser("merge")
90+
merge_parser.add_argument("source", type=Path)
91+
merge_parser.add_argument("destination", type=Path)
92+
args = parser.parse_args()
93+
if args.command == "sync":
94+
sync(args.source, args.destination, args.language)
95+
else:
96+
merge(args.source, args.destination)
97+
98+
99+
if __name__ == "__main__":
100+
main()

‎scripts/update.sh‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ set -e
77
# Allow language being passed as 1st argument, defaults to ja
88
LANGUAGE=${1:-ja}
99

10-
ROOTDIR=$(realpath "$(dirname $0)/..")
10+
ROOTDIR=$(realpath "$(dirname "$0")/..")
1111

1212
cd "${ROOTDIR}"
1313

@@ -16,21 +16,23 @@ if ! test -f cpython/Doc/conf.py; then
1616
exit 1
1717
fi
1818

19+
# Use fresh catalogs so renamed or removed resources cannot survive a prior run.
20+
mkdir -p cpython/Doc/locales
21+
WORKDIR=$(mktemp -d "${ROOTDIR}/cpython/Doc/locales/update.XXXXXX")
22+
trap 'rm -rf "$WORKDIR"' EXIT
23+
trap 'exit 1' HUP INT TERM
24+
1925
# Create POT Files
2026
cd cpython/Doc
21-
sphinx-build -E -b gettext -D gettext_compact=0 -d build/.doctrees . locales/pot
27+
sphinx-build -E -b gettext -D gettext_compact=0 -d "${WORKDIR}/.doctrees" . "${WORKDIR}/pot"
2228

23-
# Update CPython's .tx/config
24-
cd locales
29+
# Generate a fresh .tx/config from the current POT files.
30+
cd "${WORKDIR}"
2531
sphinx-intl create-txconfig
2632
sphinx-intl update-txconfig-resources -p pot -d . --transifex-organization-name python-doc --transifex-project-name python-newest
27-
install -Dm644 .tx/config "${ROOTDIR}/.tx/config"
2833

29-
# Pull translations into cpython/Doc/locales/LANGUAGE/LC_MESSAGES/
30-
tx pull -l ${LANGUAGE} -t --use-git-timestamps
34+
# Pull translations into the temporary LANGUAGE/LC_MESSAGES directory.
35+
tx pull -l "${LANGUAGE}" -t --use-git-timestamps
3136

32-
# Finally, move downloaded translation files to the language's repository
33-
cd "${LANGUAGE}/LC_MESSAGES/"
34-
for po in $(find . -type f -name '*.po' | sort | sed 's|^\./||'); do
35-
install -Dm644 ${po} "${ROOTDIR}/${po}"
36-
done
37+
# Validate the download before copying catalogs or removing obsolete files.
38+
python "${ROOTDIR}/scripts/translations.py" sync "${WORKDIR}" "${ROOTDIR}" --language "${LANGUAGE}"

‎tests/test_translations.py‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
import json
2+
from pathlib import Path
3+
import shutil
4+
import subprocess
5+
import sys
6+
import tempfile
7+
import unittest
8+
9+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
10+
import translations
11+
12+
13+
def write_config(root, catalogs):
14+
path = root / ".tx/config"
15+
path.parent.mkdir(parents=True, exist_ok=True)
16+
text = "[main]\nhost = https://www.transifex.com\n"
17+
for index, catalog in enumerate(catalogs):
18+
template = Path(catalog).with_suffix(".pot").as_posix()
19+
text += f"\n[resource{index}]\nsource_file = pot/{template}\n"
20+
path.write_text(text, encoding="utf-8")
21+
22+
23+
def write_po(root, name, translation):
24+
path = root / name
25+
path.parent.mkdir(parents=True, exist_ok=True)
26+
path.write_text(
27+
'msgid ""\nmsgstr "Content-Type: text/plain; charset=UTF-8\\n"\n\n'
28+
'msgid "example"\nmsgstr ' + json.dumps(translation, ensure_ascii=False) + "\n",
29+
encoding="utf-8",
30+
)
31+
return path
32+
33+
34+
class TranslationTests(unittest.TestCase):
35+
def setUp(self):
36+
temporary = tempfile.TemporaryDirectory()
37+
self.addCleanup(temporary.cleanup)
38+
self.root = Path(temporary.name)
39+
self.source = self.root / "source"
40+
self.destination = self.root / "destination"
41+
self.source.mkdir()
42+
self.destination.mkdir()
43+
subprocess.run(
44+
["git", "init", "--quiet", str(self.destination)], check=True
45+
)
46+
47+
def track(self):
48+
subprocess.run(
49+
["git", "-C", str(self.destination), "add", "."], check=True
50+
)
51+
52+
def test_sync_removes_old_path_and_preserves_untracked_files(self):
53+
catalogs = ["builtins/functions.po", "library/builtins.po", "bugs.po"]
54+
write_config(self.source, catalogs)
55+
download = self.source / "ja/LC_MESSAGES"
56+
for path in catalogs:
57+
write_po(download, path, "新しい訳")
58+
# A stale downloaded file must not be copied back into the repository.
59+
write_po(download, "library/functions.po", "古い訳")
60+
old = write_po(self.destination, "library/functions.po", "古い訳")
61+
write_po(self.destination, "library/builtins.po", "モジュール")
62+
self.track()
63+
scratch = write_po(self.destination, "scratch.po", "作業中")
64+
scratch_before = scratch.read_bytes()
65+
66+
translations.sync(self.source, self.destination, "ja")
67+
68+
self.assertFalse(old.exists())
69+
for path in catalogs:
70+
self.assertEqual(
71+
(download / path).read_bytes(),
72+
(self.destination / path).read_bytes(),
73+
)
74+
self.assertEqual(scratch_before, scratch.read_bytes())
75+
self.assertEqual(
76+
(self.source / ".tx/config").read_bytes(),
77+
(self.destination / ".tx/config").read_bytes(),
78+
)
79+
80+
def test_incomplete_download_does_not_change_repository(self):
81+
write_config(self.source, ["builtins/functions.po", "bugs.po"])
82+
write_po(self.source / "ja/LC_MESSAGES", "builtins/functions.po", "新しい訳")
83+
write_config(self.destination, ["library/functions.po"])
84+
old = write_po(self.destination, "library/functions.po", "古い訳")
85+
self.track()
86+
before = old.read_bytes()
87+
config_before = (self.destination / ".tx/config").read_bytes()
88+
89+
with self.assertRaises(FileNotFoundError):
90+
translations.sync(self.source, self.destination, "ja")
91+
92+
self.assertEqual(before, old.read_bytes())
93+
self.assertEqual(config_before, (self.destination / ".tx/config").read_bytes())
94+
self.assertFalse((self.destination / "builtins/functions.po").exists())
95+
96+
def test_empty_config_does_not_delete_catalogs(self):
97+
write_config(self.source, [])
98+
old = write_po(self.destination, "library/functions.po", "古い訳")
99+
self.track()
100+
with self.assertRaises(ValueError):
101+
translations.sync(self.source, self.destination, "ja")
102+
self.assertTrue(old.exists())
103+
104+
def test_config_cannot_escape_catalog_directory(self):
105+
write_config(self.source, ["../outside.po"])
106+
with self.assertRaises(ValueError):
107+
translations.active_catalogs(self.source / ".tx/config")
108+
109+
@unittest.skipUnless(shutil.which("pomerge"), "pomerge is required")
110+
def test_merge_uses_current_translation_across_renamed_paths(self):
111+
write_config(self.source, ["builtins/functions.po"])
112+
write_po(self.source, "builtins/functions.po", "新しい訳")
113+
write_po(self.source, "library/functions.po", "古い訳")
114+
old = write_po(self.destination, "library/functions.po", "以前の訳")
115+
self.track()
116+
117+
translations.merge(self.source, self.destination)
118+
119+
text = old.read_text(encoding="utf-8")
120+
self.assertIn('msgstr "新しい訳"', text)
121+
self.assertNotIn("古い訳", text)
122+
self.assertFalse((self.destination / "builtins/functions.po").exists())
123+
124+
125+
if __name__ == "__main__":
126+
unittest.main()

0 commit comments

Comments
 (0)