diff --git a/.github/release-contract/java-ui.json b/.github/release-contract/java-ui.json new file mode 100644 index 000000000..44cc54dd8 --- /dev/null +++ b/.github/release-contract/java-ui.json @@ -0,0 +1,14 @@ +{ + "schema": 1, + "coordinates": "ly.count.sdk:java-ui", + "pomDependencies": [ + "ly.count.sdk:java:{version}:compile", + "org.json:json:20250517:runtime", + "org.openjfx:javafx-controls:21.0.5:runtime", + "org.openjfx:javafx-web:21.0.5:runtime" + ], + "moduleVariantAttributes": { + "org.gradle.jvm.version": 17 + }, + "maxClassFileMajor": 61 +} diff --git a/.github/release-contract/java.json b/.github/release-contract/java.json new file mode 100644 index 000000000..755a1f3d4 --- /dev/null +++ b/.github/release-contract/java.json @@ -0,0 +1,12 @@ +{ + "schema": 1, + "coordinates": "ly.count.sdk:java", + "pomDependencies": [ + "com.google.code.findbugs:jsr305:3.0.2:runtime", + "org.json:json:20250517:runtime" + ], + "moduleVariantAttributes": { + "org.gradle.jvm.version": 8 + }, + "maxClassFileMajor": 52 +} diff --git a/.github/release/.gitignore b/.github/release/.gitignore new file mode 100644 index 000000000..4dd207434 --- /dev/null +++ b/.github/release/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +consumers/maven/target/ diff --git a/.github/release/checks.py b/.github/release/checks.py new file mode 100644 index 000000000..7ec668b6b --- /dev/null +++ b/.github/release/checks.py @@ -0,0 +1,22 @@ +"""Checks on the tag commit: its release branch and the required status checks.""" + +import subprocess + + +def branch_problems(commit, branch, repo_root, runner=subprocess.run): + """Problems unless the commit is an ancestor of origin/.""" + result = runner(["git", "-C", str(repo_root), "merge-base", "--is-ancestor", commit, f"origin/{branch}"], capture_output=True) + return [] if result.returncode == 0 else [f"{commit} is not on origin/{branch}; the tag must point to that branch"] + + +def missing_checks(check_runs, required): + """Required check names that did not succeed on the commit, from the GitHub check-runs API response: a name without + any run, or with any run that did not succeed or has not finished. A commit can carry several runs of one check, + for example one from the push to the release branch and one from a pull request whose head it is, which tests a + merge with another branch; each of them must have passed.""" + problems = [] + for name in required: + conclusions = [run.get("conclusion") for run in check_runs.get("check_runs", []) if run["name"] == name] + if not conclusions or any(conclusion != "success" for conclusion in conclusions): + problems.append(name) + return problems diff --git a/.github/release/config.json b/.github/release/config.json new file mode 100644 index 000000000..05b08de27 --- /dev/null +++ b/.github/release/config.json @@ -0,0 +1,33 @@ +{ + "repository": "Countly/countly-sdk-java", + "publicBaseUrl": { "production": "https://maven.countly.com/", "test": "https://maven-test.countly.com/" }, + "centralBaseUrl": "https://repo1.maven.org/maven2/", + "publishedModules": ":sdk-java,:sdk-java-ui", + "platformSpecificGroups": ["org.openjfx"], + "requiredChecks": ["Build and Test", "Test on ubuntu-latest", "Test on windows-latest", "OSV scan (all modules)", "Analyze (java)"], + "tagRules": [ + { "tag": "{version}", "branch": "staging", "artifacts": ["java", "java-ui"] } + ], + "artifacts": { + "java": { + "group": "ly.count.sdk", "artifact": "java", "module": ":sdk-java", + "contract": ".github/release-contract/java.json", "consumerJava": 8, + "versionSources": [ + { "file": "gradle.properties", "regex": "^VERSION_NAME=(.+)$" }, + { "file": "build.gradle", "regex": "ext\\.CLY_VERSION = \"([^\"]+)\"" }, + { "file": "sdk-java/src/main/java/ly/count/sdk/java/Config.java", "regex": "protected String sdkVersion = \"([^\"]+)\"" } + ], + "changelog": "CHANGELOG.md" + }, + "java-ui": { + "group": "ly.count.sdk", "artifact": "java-ui", "module": ":sdk-java-ui", + "contract": ".github/release-contract/java-ui.json", "consumerJava": 17, + "versionSources": [ + { "file": "gradle.properties", "regex": "^VERSION_NAME=(.+)$" }, + { "file": "build.gradle", "regex": "ext\\.CLY_VERSION = \"([^\"]+)\"" }, + { "file": "sdk-java/src/main/java/ly/count/sdk/java/Config.java", "regex": "protected String sdkVersion = \"([^\"]+)\"" } + ], + "changelog": "CHANGELOG.md" + } + } +} diff --git a/.github/release/consumers.py b/.github/release/consumers.py new file mode 100644 index 000000000..5f480c9f8 --- /dev/null +++ b/.github/release/consumers.py @@ -0,0 +1,50 @@ +"""Commands for the clean projects in .github/release/consumers that build against a staged release.""" + +from pathlib import Path + +GRADLE_PROJECT = ".github/release/consumers/gradle" +MAVEN_POM = ".github/release/consumers/maven/pom.xml" +GRADLE_FLAGS = ["--no-daemon", "--no-configuration-cache", "--stacktrace"] +PROBE_CLASS = "ly.count.consumer.Probe" +REPOSITORY_ID = "countly" + + +def gradle_command(gradlew, repository, artifact, version): + """Builds the Gradle consumer against one artifact version and installs it with its runtime classpath.""" + return [ + gradlew, "-p", GRADLE_PROJECT, *GRADLE_FLAGS, "clean", "installDist", + f"-PcountlyRepository={repository}", f"-PcountlyDependency={artifact.coordinates}:{version}", + f"-PcountlyRelease={artifact.consumer_java}", f"-PcountlyProbe={artifact.artifact}", + ] + + +def maven_command(mvn, repository, artifact, version, local_repository): + """Compiles the Maven consumer against one artifact version, with an empty local repository so nothing is reused.""" + return [ + mvn, "-B", "-f", MAVEN_POM, "clean", "compile", + f"-Dcountly.repository={repository}", f"-Dcountly.artifact={artifact.artifact}", f"-Dcountly.version={version}", + f"-Dcountly.release={artifact.consumer_java}", f"-Dmaven.repo.local={local_repository}", + ] + + +def smoke_command(java_home, repo_root): + """Runs the probe the Gradle consumer installed, on the given Java runtime, with the installed runtime classpath.""" + java = Path(java_home) / "bin" / "java" + classpath = Path(repo_root) / GRADLE_PROJECT / "build/install/countly-consumer/lib/*" + return [str(java), "-cp", str(classpath), PROBE_CLASS] + + +def maven_source_problems(local_repository, plan, consumed): + """Problems when Maven did not take the consumed artifact, and every other artifact of the plan it downloaded, from + the staged release; Maven records the source repository of every downloaded file in _remote.repositories.""" + problems = [] + for artifact in plan.artifacts: + record = Path(local_repository) / artifact.folder(plan.version) / "_remote.repositories" + if not record.is_file(): + if artifact == consumed: + problems.append(f"Maven did not download {artifact.coordinates}:{plan.version}") + continue + sources = [line.split(">", 1)[1] for line in record.read_text(encoding="utf-8").splitlines() if ">" in line and not line.startswith("#")] + if not sources or any(source != f"{REPOSITORY_ID}=" for source in sources): + problems.append(f"Maven resolved {artifact.coordinates}:{plan.version} from another repository than the staged release") + return problems diff --git a/.github/release/consumers/gradle/build.gradle b/.github/release/consumers/gradle/build.gradle new file mode 100644 index 000000000..e9075b4fe --- /dev/null +++ b/.github/release/consumers/gradle/build.gradle @@ -0,0 +1,20 @@ +plugins { + id 'java' + id 'application' +} + +def probe = findProperty('countlyProbe') + +sourceSets.main.java.srcDirs = ["../probes/${probe}"] + +tasks.withType(JavaCompile).configureEach { + options.release = Integer.parseInt(findProperty('countlyRelease')) +} + +dependencies { + implementation findProperty('countlyDependency') +} + +application { + mainClass = 'ly.count.consumer.Probe' +} diff --git a/.github/release/consumers/gradle/settings.gradle b/.github/release/consumers/gradle/settings.gradle new file mode 100644 index 000000000..b663fe4e9 --- /dev/null +++ b/.github/release/consumers/gradle/settings.gradle @@ -0,0 +1,16 @@ +def countlyRepository = gradle.startParameter.projectProperties['countlyRepository'] + +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + exclusiveContent { + forRepository { + maven { url = countlyRepository } + } + filter { includeGroupByRegex 'ly\\.count\\..*' } + } + mavenCentral() + } +} + +rootProject.name = 'countly-consumer' diff --git a/.github/release/consumers/maven/pom.xml b/.github/release/consumers/maven/pom.xml new file mode 100644 index 000000000..931a459c8 --- /dev/null +++ b/.github/release/consumers/maven/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + ly.count.consumer + countly-consumer + 1 + + + UTF-8 + ${countly.release} + + + + + countly + ${countly.repository} + + + + + + ly.count.sdk + ${countly.artifact} + ${countly.version} + + + + + ${project.basedir}/../probes/${countly.artifact} + + + org.apache.maven.plugins + maven-compiler-plugin + 3.14.0 + + + + diff --git a/.github/release/consumers/probes/java-ui/ly/count/consumer/Probe.java b/.github/release/consumers/probes/java-ui/ly/count/consumer/Probe.java new file mode 100644 index 000000000..b4ea458b1 --- /dev/null +++ b/.github/release/consumers/probes/java-ui/ly/count/consumer/Probe.java @@ -0,0 +1,18 @@ +package ly.count.consumer; + +import ly.count.sdk.java.ui.CountlyWebView; +import ly.count.sdk.java.ui.JavaFxContentDisplay; + +/** + * Compiles against the UI artifact the way an integrator would. + */ +public class Probe { + /** + * Reads a setting of the UI artifact and names its content display class. + * + * @param args unused + */ + public static void main(String[] args) { + System.out.println("widgets within app " + CountlyWebView.isShowingWidgetsWithinApp() + ", display " + JavaFxContentDisplay.class.getName()); + } +} diff --git a/.github/release/consumers/probes/java/ly/count/consumer/Probe.java b/.github/release/consumers/probes/java/ly/count/consumer/Probe.java new file mode 100644 index 000000000..792838993 --- /dev/null +++ b/.github/release/consumers/probes/java/ly/count/consumer/Probe.java @@ -0,0 +1,20 @@ +package ly.count.consumer; + +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; + +/** + * Loads the core SDK the way an integrator would, without contacting a server. + */ +public class Probe { + /** + * Creates a configuration and reads the shared instance; a class the runtime cannot load stops here. + * + * @param args unused + */ + public static void main(String[] args) { + Config config = new Config("https://consumer.invalid", "consumer-probe"); + System.out.println("Countly " + config.getSdkVersion() + " loaded on Java " + System.getProperty("java.version") + + ", instance " + Countly.instance().getClass().getName() + ", initialized " + Countly.isInitialized()); + } +} diff --git a/.github/release/contract.py b/.github/release/contract.py new file mode 100644 index 000000000..26afb4717 --- /dev/null +++ b/.github/release/contract.py @@ -0,0 +1,126 @@ +"""Compares what a staged artifact promises integrators with its committed contract file.""" + +import io +import json +import zipfile +import xml.etree.ElementTree as ElementTree +from pathlib import Path + +POM_NAMESPACE = {"m": "http://maven.apache.org/POM/4.0.0"} +JVM_VERSION = "org.gradle.jvm.version" +RELEASE_VERSION = "{version}" + + +def _child(element, tag, default=""): + return (element.findtext(f"m:{tag}", default, POM_NAMESPACE) or default).strip() + + +def pom_dependencies(pom): + """Sorted group:artifact:version:scope strings of the POM's direct dependencies.""" + root = ElementTree.fromstring(pom) + return sorted( + f"{_child(d, 'groupId')}:{_child(d, 'artifactId')}:{_child(d, 'version')}:{_child(d, 'scope', 'compile')}" + for d in root.findall("m:dependencies/m:dependency", POM_NAMESPACE) + ) + + +def release_dependencies(dependencies, group, version): + """Dependencies with the version of the release itself written as {version} for artifacts of the same group (java-ui + depends on java of its own release), so a contract holds for every release.""" + shown = [] + for dependency in dependencies: + parts = dependency.split(":") + if parts[0] == group and parts[2] == version: + parts[2] = RELEASE_VERSION + shown.append(":".join(parts)) + return shown + + +def module_variants(module): + """Library variants (not sources or javadoc) of a Gradle module file.""" + return [variant for variant in json.loads(module).get("variants", []) if variant.get("attributes", {}).get("org.gradle.category") == "library"] + + +def module_dependencies(module): + """Sorted group:module:version strings declared by the library variants of a Gradle module file.""" + found = set() + for variant in module_variants(module): + for dependency in variant.get("dependencies", []): + version = dependency.get("version", {}) + number = version.get("requires") or version.get("strictly") or version.get("prefers") or "" + found.add(f"{dependency['group']}:{dependency['module']}:{number}") + return sorted(found) + + +def max_class_major(jar): + """Highest class-file major version in a jar (52 is Java 8), ignoring multi-release folders.""" + highest = 0 + with zipfile.ZipFile(io.BytesIO(jar)) as archive: + for name in archive.namelist(): + if name.endswith(".class") and not name.startswith("META-INF/versions/"): + head = archive.read(name)[:8] + if len(head) == 8 and head[:4] == b"\xca\xfe\xba\xbe": + highest = max(highest, (head[6] << 8) | head[7]) + return highest + + +def _common(variants, key): + """The attribute value all variants share (None when absent), or a note that they differ.""" + values = {json.dumps(variant.get("attributes", {}).get(key)) for variant in variants} + if len(values) == 1: + return json.loads(values.pop()) + return None if not values else "differs between variants" + + +def actual_contract(staging_dir, artifact, version): + """The contract the staged files of one artifact version fulfil, in the shape of the contract files (schema 1).""" + folder = Path(staging_dir) / artifact.folder(version) + base = artifact.base_name(version) + return { + "schema": 1, + "coordinates": artifact.coordinates, + "pomDependencies": release_dependencies(pom_dependencies((folder / f"{base}.pom").read_bytes()), artifact.group, version), + "moduleVariantAttributes": {JVM_VERSION: _common(module_variants((folder / f"{base}.module").read_bytes()), JVM_VERSION)}, + "maxClassFileMajor": max_class_major((folder / f"{base}.jar").read_bytes()), + } + + +def compare(expected, actual): + """Differences between the committed contract and the staged artifact, as readable problems.""" + problems = [] + if expected["coordinates"] != actual["coordinates"]: + problems.append(f"coordinates are {actual['coordinates']}, the contract says {expected['coordinates']}") + if expected["pomDependencies"] != actual["pomDependencies"]: + problems.append(f"POM dependencies are {actual['pomDependencies']}, the contract says {expected['pomDependencies']}") + for key, value in expected.get("moduleVariantAttributes", {}).items(): + found = actual["moduleVariantAttributes"].get(key) + if found != value: + problems.append(f"{key} is {found}, the contract says {value}") + if actual["maxClassFileMajor"] > expected["maxClassFileMajor"]: + problems.append(f"class files reach major version {actual['maxClassFileMajor']}, the contract allows {expected['maxClassFileMajor']}") + return problems + + +def module_dependency_problems(staging_dir, artifact, version): + """Problems when the .module declares other dependencies than the POM; Gradle builds read the .module, Maven builds the POM.""" + folder = Path(staging_dir) / artifact.folder(version) + base = artifact.base_name(version) + in_module = module_dependencies((folder / f"{base}.module").read_bytes()) + in_pom = sorted(entry.rsplit(":", 1)[0] for entry in pom_dependencies((folder / f"{base}.pom").read_bytes())) + return [] if in_module == in_pom else [f"the .module declares {in_module} but the POM declares {in_pom}"] + + +def _shown(value): + return "absent" if value is None else str(value) + + +def summary_rows(expected, actual): + """(property, contract, built) rows for the job summary table.""" + def listing(values): + return ", ".join(values) if values else "none" + + return [ + ("POM dependencies", listing(expected["pomDependencies"]), listing(actual["pomDependencies"])), + (JVM_VERSION, _shown(expected.get("moduleVariantAttributes", {}).get(JVM_VERSION)), _shown(actual["moduleVariantAttributes"].get(JVM_VERSION))), + ("highest class file version", f"at most {expected['maxClassFileMajor']}", str(actual["maxClassFileMajor"])), + ] diff --git a/.github/release/countly-sdk-signing.asc b/.github/release/countly-sdk-signing.asc new file mode 100644 index 000000000..3d024de9d --- /dev/null +++ b/.github/release/countly-sdk-signing.asc @@ -0,0 +1,54 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Comment: Hostname: +Version: Hockeypuck 2.2 + +xsFNBGCBvh0BEADYg3FP0080Vx4RfDvWkC7TfqhhypIR3yBOrIomLLLatsWAnSLO +wOm0HyfldcgrAEK8pm+i2G65QorLKep4nmL5UQ44CU+iEeiEW5obP3Wdw4teA+ra +DNTQ221i8tVQcGYVQwWY1pS0Zkh6ZrEjuSZysm5klABAk1qdh640FB7jX6WUPh28 +vRO4HRC5bSOaL5INyV/tSOhemHqhw1WI8euWFMjIaOAWrI7sMqGl4doxSdSWz04P +JyarHOlwwQxt/PRFWWF+Vkm/ByuAYpp4/yYvUHJBohwwjV+CS2Ouq4rfMfpx1Y67 +1nv7ihiE4rAnmy2QQS0R8DvVWEfesfRldkb7k4XxNbKuOBTvfUozZx9yiNIq3HhI +5qtni/rVadgtZRre051rfJDXFFajEL1bLWPvJFasH1S3BM6n6BmALdkQ9mKvAh3w +mcxWCzmlK89d+QKRXvdq5t6+PTkoOcvWhLiRF0QUwE0SW8yYUQRXMQuIta7mZFvU +P3urgM075fdO90GgZYrFyaUiGFXlBEhM3pZ41ohywVnXJIbP8AcY7xvCb2B2sBfq +75IVye4lH9dC7GZeVUODU1MiHPZ6IHuxfM9ZgUgP2ZzQrbZUHS3JnWx22FIaeatz +LNIXeoorf+Y8Plc6CaLargGWgNQtZJ9jiPKtUDrzza9cZvjOS69LJnuOJQARAQAB +zStBcnR1cnMgS2FkaWtpcyAoQW5kcm9pZCBTREspIDxha2FAY291bnQubHk+wsGO +BBMBCAA4FiEE/I2w3iNKJzukXlYvuMg6B5pbvQwFAmCBvh0CGwMFCwkIBwIGFQoJ +CAsCBBYCAwECHgECF4AACgkQuMg6B5pbvQz+gBAAzDEQGN8PLsGCT9rh9489LFSq +HUtODrHYE2RQPA1eA9IrYpX6jOWsgQ+qomI0aZoVO1PJZ7uLDUFFa4h//+jGDVcL +8xtKPmknGl+ercg6wqPDt/q3NKi5BvD4klHgzxarKasq65Mhcn1izyi9n3lvNza/ +4xezW7NGU6EiIrEH9Rp1r5wxy3+axS7TLlJar+wdUc/U1OkoPuoar+2xK8peRJRb +pI5KGcvZo6XodNRJK5e+L1MWVvQsJM8kknO+9k6NmKS+R6F9mjqcO0OOYoVulPK8 +DwKFYieyNq3SKSpvzDof66eTeaV3JQOnukj7pgRvaUmNOzbsGCTIG4tdK/2VlgmQ +AWPZCJvipW8zFWDxsTiOq6SnmOG0cQ1zYRYvg91lNrsFOROpvSKfmWcOlTi0xlkU +A6lTPXJ/8+c1aFHRB7SoBF9FAjNTXXjhGRraTH5scyW76exOmLDmMloQSrUvIOBk +qiEJl4NiYrYIcMHxy3P9x1TiZSmwjDRbkPDF2+WVuxlXuz844NTnrErpXlphWC2z +LDo5n+uO2BWkh0Acp+CBRg1P0naZUHcbX2kyXCePvhemsB4WVBp596KzfWj0JN8k +iLL5+oM2JJ9TnE0LxepvaFYxi2E5F9qOLtzKhqS/ENrL7tPqnGk2yJY9X/5eh2qV +uEttgau92EdSsNZomS/OwU0EYIG+HQEQALypyd4svykhkPgf1cuoTn/bQFBKZt06 +fkxfy/A5btcXuzPjJxc8kIpBHC4TM8wwtYOBjqrgqPcaEqlWfDpF8DjsnGEFyjC/ +xSPy7DO8huQHYiQhqk8di270Uaqh+sMCKPn9Y2qVB6aPdTTO269nL4gJhRuwyqpd +KUc20jVLFV2BkyYWY9gZiXIgvfdYoX7brJJAZJmggK/5D+eZLv7oO3j8doJiMPAY +tIJghgTwvxAHPG5ANmbB50ie8TzjER2zGpYFFi5byXR1RrwIXh0DN3YRnJln3xyT +kzbUIukSXdlEt1tglmDHFGNiDxYKmwNSthdtuVExfYE521j2XEW5fHHYcKAVI9gN +9vA3Ut/xU8ycK8baFesLIeXe1ycc1PytLJA8T7yUT7FPmh0xdL9KNM1lrCJtHT/Z +jdFcjdTavE7TGwDN9apHyqgZxvAmcuzjGhfexmoJ7arIWMaDehA52jajuUFLiUqR +Rum6XzihS24RN5E+5hEztmdNKwh/14KJsEK5d+/ZMm8hJa0jFHd0kPXRUfKLb2cX +H/XADBbzUZ6fqtp6rBVJUJOcuQ3DYt4W4Ejov+d/UxGbt3PxUSbX8DOlEs0l2ckf +lqAkBuOuU+63zTXdNnYsg+NAmZ6mtDbjCYTmE8sOugVjGwRslP7ckZRxvMRNFSUU +KfokrCn72ArRABEBAAHCwXYEGAEIACAWIQT8jbDeI0onO6ReVi+4yDoHmlu9DAUC +YIG+HQIbDAAKCRC4yDoHmlu9DCs5D/sGHF8dcgI929Hrv2zy0KiSjlw7+Eft1Z8Z +NABEoB0ZZwnIb2KD7O4CYcPDTuF/AKGZlxa8nrHplY8YLMHba+u1OFxfCMrNaf3X +f+VESPTEJB18+myw3IeMO2CCNwsh6z0rika6HYZLy2d8G3tiF224gPvBwvOpbAYd +EW4gFPFUP34k96ToL7MEYfHi1TMvfzKg7Ggw8qUbfrbsTK20/3YdkZaIzvsWqcVT +tIjycMPq7+I7j+Bii1pOUkQIB1iQB4ijITm5WswYiy74KMK6D0juPqXBY9m6LQ37 +5cTq9/7/1xGXB+JFi3xLEl/Vccfe5dnlWaZyt4SzRYl9uu5CtO7Yh3rQrbjqb298 +LxkqbHEm1wWMp1vPWa35v15t5hNCzh2HIkH6S+xMUgE9ZhCfNNkJrZtZpzSw6Uyn +vtqdymMTIxF5NA/zj6wwiSlGEkRA5dFeO1B140aRRHsBEBpYTGpFLJRxG2mhNmt3 +o6fiasg8fgMtcvawSR6BkqcN9iDMkDlMMJetaSmS8ITyAv5cAaUa1/grYw4s6LvV +QgkXuFsVkCX9biffcAYibRYhXt+jHwP5QwTvFgfuijiQBRna1HwUWYhDwtz5mjZ2 +WKfeJCvcxVveJd31Oqdip3eSJ9QQA0CTBL0QyNL+ZL1KVQAqlptRQKGFk0V4tK3/ +csSJEAExRA== +=THAH +-----END PGP PUBLIC KEY BLOCK----- diff --git a/.github/release/index.py b/.github/release/index.py new file mode 100644 index 000000000..ab714f413 --- /dev/null +++ b/.github/release/index.py @@ -0,0 +1,73 @@ +"""Rewrites maven-metadata.xml of an artifact from the complete version folders in the bucket.""" + +import hashlib +import re +import tempfile +from pathlib import Path +from xml.sax.saxutils import escape + +from layout import CHECKSUM_ALGORITHMS +from plan import FINAL_VERSION + +METADATA_CACHE = "public, max-age=60, must-revalidate" + + +def complete_versions(keys, group_path, artifact_name): + """Versions whose folder holds the POM; the POM is always uploaded last, so its presence means complete.""" + prefix = f"{group_path}/{artifact_name}/" + versions = set() + for key in keys: + if key.startswith(prefix): + parts = key[len(prefix):].split("/") + if len(parts) == 2 and parts[1] == f"{artifact_name}-{parts[0]}.pom": + versions.add(parts[0]) + return versions + + +def listed_versions(versions): + """Versions that belong in the index, in ascending numeric order: final versions only, so release candidates and + any folder the release workflow cannot produce stay unlisted.""" + final = [version for version in versions if re.fullmatch(FINAL_VERSION, version, re.ASCII)] + return sorted(final, key=lambda version: tuple(int(part) for part in version.split("."))) + + +def render(group, artifact_name, versions, now): + """maven-metadata.xml for ascending versions; latest and release are the highest version.""" + highest = escape(versions[-1]) + return "\n".join([ + '', + "", + f" {escape(group)}", + f" {escape(artifact_name)}", + " ", + f" {highest}", + f" {highest}", + " ", + *[f" {escape(version)}" for version in versions], + " ", + f" {now.strftime('%Y%m%d%H%M%S')}", + " ", + "", + "", + ]) + + +def rewrite_index(bucket, group, artifact_name, now, log=print): + """Rewrites the index from the bucket: the four checksum files first, the body last. Returns the listed versions.""" + group_path = group.replace(".", "/") + versions = listed_versions(complete_versions(bucket.list_keys(f"{group_path}/{artifact_name}/"), group_path, artifact_name)) + if not versions: + log(f"no listed version of {group}:{artifact_name}; index left unchanged") + return [] + body = render(group, artifact_name, versions, now).encode("utf-8") + key = f"{group_path}/{artifact_name}/maven-metadata.xml" + with tempfile.TemporaryDirectory() as folder: + for suffix, algorithm in CHECKSUM_ALGORITHMS: + sidecar = Path(folder) / f"maven-metadata.xml.{suffix}" + sidecar.write_bytes(hashlib.new(algorithm, body).hexdigest().encode("ascii")) + bucket.put_file(f"{key}.{suffix}", sidecar, "text/plain; charset=utf-8", METADATA_CACHE, create_only=False) + main = Path(folder) / "maven-metadata.xml" + main.write_bytes(body) + bucket.put_file(key, main, "application/xml", METADATA_CACHE, create_only=False) + log(f"index {key}: {len(versions)} versions, latest {versions[-1]}") + return versions diff --git a/.github/release/layout.py b/.github/release/layout.py new file mode 100644 index 000000000..34d8ccb3e --- /dev/null +++ b/.github/release/layout.py @@ -0,0 +1,64 @@ +"""Expected files of a release in the staging folder, and checksum helpers.""" + +import hashlib +from pathlib import Path + +CHECKSUM_ALGORITHMS = (("md5", "md5"), ("sha1", "sha1"), ("sha256", "sha256"), ("sha512", "sha512")) +CHECKSUM_SUFFIXES = tuple("." + suffix for suffix, _ in CHECKSUM_ALGORITHMS) + + +def primary_files(artifact, version): + """Primary file names of one artifact version, before checksums and signatures.""" + base = artifact.base_name(version) + return [f"{base}.jar", f"{base}.pom", f"{base}.module", f"{base}-sources.jar", f"{base}-javadoc.jar", f"{base}-cyclonedx.json"] + + +def expected_files(artifact, version): + """Every file name expected in the version folder: the primary files and their four checksums.""" + return sorted(name + suffix for name in primary_files(artifact, version) for suffix in ("",) + CHECKSUM_SUFFIXES) + + +def is_checksum(name): + """True for .md5, .sha1, .sha256 and .sha512 files.""" + return name.endswith(CHECKSUM_SUFFIXES) + + +def write_checksums(path): + """Writes path.md5, .sha1, .sha256 and .sha512 as lowercase hex without a file name, like Maven Central.""" + data = Path(path).read_bytes() + for suffix, algorithm in CHECKSUM_ALGORITHMS: + Path(f"{path}.{suffix}").write_bytes(hashlib.new(algorithm, data).hexdigest().encode("ascii")) + + +def remove_index_files(staging_dir): + """Deletes the maven-metadata.xml files Gradle writes; the publish job writes the real index.""" + for path in Path(staging_dir).rglob("maven-metadata.xml*"): + path.unlink() + + +def check_staging(staging_dir, plan): + """Problems with the staging folder: missing or unexpected files for the tag's artifacts, wrong checksums.""" + root = Path(staging_dir) + problems = [] + expected_paths = set() + for artifact in plan.artifacts: + relative = artifact.folder(plan.version) + folder = root / relative + expected = expected_files(artifact, plan.version) + expected_paths.update(f"{relative}/{name}" for name in expected) + present = sorted(path.name for path in folder.iterdir()) if folder.is_dir() else [] + problems += [f"missing {relative}/{name}" for name in expected if name not in present] + problems += [f"unexpected {relative}/{name}" for name in present if name not in expected] + for name in primary_files(artifact, plan.version): + path = folder / name + if not path.is_file(): + continue + data = path.read_bytes() + for suffix, algorithm in CHECKSUM_ALGORITHMS: + sidecar = folder / f"{name}.{suffix}" + if sidecar.is_file() and sidecar.read_bytes().decode("ascii").strip() != hashlib.new(algorithm, data).hexdigest(): + problems.append(f"wrong checksum {relative}/{name}.{suffix}") + for path in root.rglob("*"): + if path.is_file() and path.relative_to(root).as_posix() not in expected_paths: + problems.append(f"unexpected {path.relative_to(root).as_posix()}") + return sorted(set(problems)) diff --git a/.github/release/manifest.py b/.github/release/manifest.py new file mode 100644 index 000000000..a4d4b97cf --- /dev/null +++ b/.github/release/manifest.py @@ -0,0 +1,28 @@ +"""The list of staged files with sizes and hashes, handed from the build job to the publish and verify jobs.""" + +import hashlib +from pathlib import Path + + +def build_manifest(staging_dir, tag, commit): + """Lists every staged file with its size and sha256.""" + root = Path(staging_dir) + files = [] + for path in sorted(p for p in root.rglob("*") if p.is_file()): + data = path.read_bytes() + files.append({"path": path.relative_to(root).as_posix(), "size": len(data), "sha256": hashlib.sha256(data).hexdigest()}) + return {"tag": tag, "commit": commit, "files": files} + + +def verify_manifest(staging_dir, manifest): + """Problems when the staging folder differs from the manifest: missing, unexpected or changed files.""" + root = Path(staging_dir) + listed = {entry["path"]: entry for entry in manifest["files"]} + present = {path.relative_to(root).as_posix() for path in root.rglob("*") if path.is_file()} + problems = [f"missing {path}" for path in sorted(set(listed) - present)] + problems += [f"unexpected {path}" for path in sorted(present - set(listed))] + for path in sorted(set(listed) & present): + data = (root / path).read_bytes() + if len(data) != listed[path]["size"] or hashlib.sha256(data).hexdigest() != listed[path]["sha256"]: + problems.append(f"changed {path}") + return problems diff --git a/.github/release/plan.py b/.github/release/plan.py new file mode 100644 index 000000000..1281f45ed --- /dev/null +++ b/.github/release/plan.py @@ -0,0 +1,118 @@ +"""Maps a release tag to what it publishes, using the repository's .github/release/config.json.""" + +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Optional, Tuple + +FINAL_VERSION = r"\d+\.\d+\.\d+" +RELEASE_VERSION = FINAL_VERSION + r"(?:-rc\d+)?" +RC_PATTERN = re.compile(r"-rc\d+$") +CONFIG_PATH = Path(__file__).resolve().parent / "config.json" + + +class PlanError(Exception): + """A tag or repository state that must stop the release.""" + + +@dataclass(frozen=True) +class Artifact: + """One Maven artifact of the repository and the Gradle module that builds it.""" + + key: str + group: str + artifact: str + module: str + contract: Optional[str] + consumer_java: int + + @property + def group_path(self): + """Group as a folder path, e.g. ly/count/sdk.""" + return self.group.replace(".", "/") + + @property + def coordinates(self): + """group:artifact.""" + return f"{self.group}:{self.artifact}" + + @property + def module_dir(self): + """Folder of the Gradle module, e.g. sdk-java-ui for :sdk-java-ui.""" + return self.module.lstrip(":").replace(":", "/") + + def folder(self, version): + """Repository folder of one version, e.g. ly/count/sdk/java/26.8.1.""" + return f"{self.group_path}/{self.artifact}/{version}" + + def base_name(self, version): + """File name stem of one version, e.g. java-26.8.1.""" + return f"{self.artifact}-{version}" + + +@dataclass(frozen=True) +class Plan: + """What one tag publishes.""" + + tag: str + version: str + listed: bool + prerelease: bool + branch: str + artifacts: Tuple[Artifact, ...] + + @property + def modules(self): + """Gradle modules to build, in order, without duplicates.""" + return list(dict.fromkeys(artifact.module for artifact in self.artifacts)) + + def to_json(self): + """Serializable form, handed between workflow jobs as plan.json.""" + return {"tag": self.tag, "version": self.version, "listed": self.listed, "prerelease": self.prerelease, "branch": self.branch, "artifacts": [artifact.key for artifact in self.artifacts]} + + +def load_config(path=CONFIG_PATH): + """Reads config.json.""" + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def artifact_from_config(config, key): + """Builds the Artifact described by config['artifacts'][key].""" + spec = config["artifacts"][key] + return Artifact(key, spec["group"], spec["artifact"], spec["module"], spec.get("contract"), spec["consumerJava"]) + + +def plan_for_tag(config, tag): + """Maps a tag to its plan; raises PlanError for a tag outside the repository's grammar (ASCII digits only).""" + for rule in config["tagRules"]: + prefix, suffix = rule["tag"].split("{version}") + match = re.fullmatch(f"{re.escape(prefix)}(?P{RELEASE_VERSION}){re.escape(suffix)}", tag, re.ASCII) + if match: + version = match.group("version") + prerelease = bool(RC_PATTERN.search(version)) + artifacts = tuple(artifact_from_config(config, key) for key in rule["artifacts"]) + return Plan(tag, version, not prerelease, prerelease, rule["branch"], artifacts) + raise PlanError(f'"{tag}" is not a release tag of {config["repository"]}') + + +def checkout_tags(config, repo_root, branch): + """Tags the checkout would be released as from the branch: one per tag rule of that branch, carrying the version + held by the first version source of the rule's first artifact. Raises PlanError when that version is unreadable.""" + tags = [] + for rule in config["tagRules"]: + if rule["branch"] != branch: + continue + source = config["artifacts"][rule["artifacts"][0]]["versionSources"][0] + path = Path(repo_root) / source["file"] + match = re.search(source["regex"], path.read_text(encoding="utf-8"), re.MULTILINE) if path.is_file() else None + if match is None: + raise PlanError(f"{source['file']}: no version found") + tags.append(rule["tag"].format(version=match.group(1).strip())) + return tags + + +def plan_from_json(config, data): + """Rebuilds a plan written by Plan.to_json.""" + artifacts = tuple(artifact_from_config(config, key) for key in data["artifacts"]) + return Plan(data["tag"], data["version"], data["listed"], data["prerelease"], data["branch"], artifacts) diff --git a/.github/release/release.py b/.github/release/release.py new file mode 100644 index 000000000..3e98d00c4 --- /dev/null +++ b/.github/release/release.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +"""Entry point of .github/workflows/release.yml: one subcommand per workflow step.""" + +import argparse +import datetime +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import checks +import consumers +import contract +import index +import layout +import manifest as manifest_module +import sbom +import sign +import sources +import upload +import verify +from plan import PlanError, checkout_tags, load_config, plan_for_tag, plan_from_json +from s3 import R2Bucket + +HERE = Path(__file__).resolve().parent +REPO_ROOT = HERE.parents[1] +PUBLIC_KEY = HERE / "countly-sdk-signing.asc" +RELEASE_KEY_FINGERPRINT = "FC8DB0DE234A273BA45E562FB8C83A079A5BBD0C" +TARGETS = ["production", "test"] + + +def summary(lines): + """Prints lines and appends them to the job summary when running in GitHub Actions.""" + for line in lines: + print(line) + path = os.environ.get("GITHUB_STEP_SUMMARY") + if path: + with open(path, "a", encoding="utf-8") as handle: + handle.write("\n".join(lines) + "\n") + + +def outputs(values): + """Writes step outputs when running in GitHub Actions.""" + path = os.environ.get("GITHUB_OUTPUT") + if path: + with open(path, "a", encoding="utf-8") as handle: + for key, value in values.items(): + handle.write(f"{key}={value}\n") + + +def finish(problems, title): + """Reports problems as error annotations and exits with 1 when there are any; reports success otherwise.""" + if problems: + for problem in problems: + print(f"::error::{problem}") + summary([f"### {title}: failed"] + [f"- {problem}" for problem in problems]) + sys.exit(1) + summary([f"### {title}: passed"]) + + +def read_plan(path): + """Loads config.json and the plan written by the plan step.""" + config = load_config() + return config, plan_from_json(config, json.loads(Path(path).read_text(encoding="utf-8"))) + + +def bucket_from_env(): + """The R2 bucket named by the R2_BUCKET and R2_ACCOUNT_ID variables.""" + return R2Bucket(os.environ["R2_BUCKET"], os.environ["R2_ACCOUNT_ID"]) + + +def staged_file(staging, artifact, version, suffix): + """Path of one staged file of an artifact version, e.g. suffix '.jar' or '-cyclonedx.json'.""" + return Path(staging) / artifact.folder(version) / f"{artifact.base_name(version)}{suffix}" + + +def cmd_plan(args): + """Maps the tag to a plan, writes plan.json and the step outputs, and checks the GitHub release type.""" + config = load_config() + try: + plan = plan_for_tag(config, args.tag) + except PlanError as error: + finish([str(error)], "Tag") + return + Path(args.out).write_bytes(json.dumps(plan.to_json(), indent=2).encode("utf-8")) + outputs({ + "environment": "maven-release" if args.target == "production" else "maven-test", + "check_tasks": " ".join(f"{module}:check" for module in plan.modules), + "publish_tasks": " ".join(f"{module}:publishAllPublicationsToReleaseStagingRepository {module}:cyclonedxDirectBom" for module in plan.modules), + "published_modules": config["publishedModules"], + }) + state = "listed" if plan.listed else "release candidate, not listed" + lines = [f"### Plan for {plan.tag}", f"- version {plan.version} ({state})", f"- branch {plan.branch}", f"- target {args.target}"] + if args.commit: + lines.append(f"- commit {args.commit}") + summary(lines + [f"- {artifact.coordinates}" for artifact in plan.artifacts]) + problems = [] + if args.target == "production": + if args.prerelease not in ("true", "false"): + problems.append("a production release needs a published GitHub release for this tag") + elif (args.prerelease == "true") != plan.prerelease: + problems.append("a tag ending in -rcN must be published as a GitHub pre-release, and a pre-release needs such a tag") + finish(problems, "GitHub release") + + +def cmd_checkout_tags(args): + """Writes the tags the checkout would be released as from --branch, as the JSON list the release checks run on.""" + try: + tags = checkout_tags(load_config(), REPO_ROOT, args.branch) + except PlanError as error: + finish([str(error)], "Checkout tags") + return + outputs({"tags": json.dumps(tags)}) + summary([f"### Tags {args.branch} would release"] + [f"- {tag}" for tag in tags or ["none"]]) + + +def cmd_check_branch(args): + """The tag commit must be on the plan's release branch.""" + _, plan = read_plan(args.plan) + finish(checks.branch_problems(args.commit, plan.branch, REPO_ROOT), "Release branch") + + +def cmd_check_sources(args): + """Version numbers and changelog headings must match the tag.""" + config, plan = read_plan(args.plan) + finish(sources.check_version_sources(REPO_ROOT, plan, config), "Version numbers") + + +def cmd_check_absent(args): + """The version must exist neither in the target repository nor, for production, on Maven Central.""" + config, plan = read_plan(args.plan) + base = config["publicBaseUrl"][args.target] + problems = [] + for artifact in plan.artifacts: + pom = f"{artifact.folder(plan.version)}/{artifact.base_name(plan.version)}.pom" + if verify.fetch(f"{base}{pom}?nocache={args.nocache}") is not None: + problems.append(f"{artifact.coordinates}:{plan.version} is already on {base}") + if args.target == "production" and verify.fetch(f"{config['centralBaseUrl']}{pom}") is not None: + problems.append(f"{artifact.coordinates}:{plan.version} is already on Maven Central") + finish(problems, "Not published yet") + + +def cmd_check_required(args): + """The required status checks must have succeeded on the tag commit.""" + config = load_config() + runs = json.loads(Path(args.check_runs).read_text(encoding="utf-8")) + finish([f"required check '{name}' did not succeed on the tag commit" for name in checks.missing_checks(runs, config["requiredChecks"])], "Required checks") + + +def cmd_stage(args): + """Removes Gradle's indexes, places the SBOMs, checks the file set and writes the manifest.""" + config, plan = read_plan(args.plan) + staging = Path(args.staging) + layout.remove_index_files(staging) + timestamp = subprocess.run(["git", "-C", str(REPO_ROOT), "show", "-s", "--format=%cI", "HEAD"], capture_output=True, text=True, check=True).stdout.strip() + problems = [] + lines = [] + modules = {artifact.module: artifact for artifact in plan.artifacts} + for artifact in plan.artifacts: + source = REPO_ROOT / artifact.module_dir / "build/reports/cyclonedx-direct/bom.json" + if not source.is_file(): + problems.append(f"no SBOM at {artifact.module_dir}/build/reports/cyclonedx-direct/bom.json") + continue + try: + placed = sbom.place_sbom(source, staging, artifact, plan.version, timestamp, config["publicBaseUrl"][args.target], modules, config["platformSpecificGroups"]) + except sbom.SbomError as error: + problems.append(str(error)) + continue + lines.append(f"- SBOM of {artifact.coordinates}: {len(json.loads(placed.read_bytes()).get('components', []))} components") + problems += layout.check_staging(staging, plan) + if not problems: + data = manifest_module.build_manifest(staging, plan.tag, args.commit) + Path(args.manifest).write_bytes(json.dumps(data, indent=2).encode("utf-8")) + lines += [f"- {len(data['files'])} files staged", "", "
Staged files", ""] + lines += [f" {entry['path']}" for entry in data["files"]] + ["", "
"] + summary(lines) + finish(problems, "Staging") + + +def cmd_validate_sbom(args): + """Validates every staged SBOM against the CycloneDX 1.6 schema with the CycloneDX CLI.""" + _, plan = read_plan(args.plan) + env = dict(os.environ, DOTNET_SYSTEM_GLOBALIZATION_INVARIANT="1") + problems = [] + for artifact in plan.artifacts: + path = staged_file(args.staging, artifact, plan.version, "-cyclonedx.json") + result = subprocess.run([args.cli, "validate", "--input-file", str(path), "--input-format", "json", "--input-version", "v1_6", "--fail-on-errors"], env=env) + if result.returncode != 0: + problems.append(f"{path.name} is not a valid CycloneDX 1.6 document") + finish(problems, "SBOM schema") + + +def cmd_contract(args): + """Compares each staged artifact with its contract file and writes the comparison table; --capture prints the actual contracts instead.""" + _, plan = read_plan(args.plan) + problems = [] + lines = [] + for artifact in plan.artifacts: + if not artifact.contract: + continue + actual = contract.actual_contract(args.staging, artifact, plan.version) + if args.capture: + print(json.dumps(actual, indent=2)) + continue + expected = json.loads((REPO_ROOT / artifact.contract).read_text(encoding="utf-8")) + found = contract.compare(expected, actual) + contract.module_dependency_problems(args.staging, artifact, plan.version) + problems += [f"{artifact.coordinates}: {problem}" for problem in found] + lines += [f"#### {artifact.coordinates}", "", "| Property | Contract | Built |", "|---|---|---|"] + lines += [f"| {name} | {wanted} | {built} |" for name, wanted, built in contract.summary_rows(expected, actual)] + [""] + if not args.capture: + summary(lines) + finish(problems, "Contract") + + +def cmd_verify_manifest(args): + """The downloaded staging folder must be exactly what the build job produced.""" + data = json.loads(Path(args.manifest).read_text(encoding="utf-8")) + finish(manifest_module.verify_manifest(args.staging, data), "Manifest") + + +def cmd_sign(args): + """Signs every staged file with the key of the target held in GNUPGHOME and checks every signature. Production + must use the release key; a dry run must use its own key. The public key the signatures verify against is written + to --public-key-out for the verify job: the committed release key, or the dry-run key exported from the keyring.""" + homedir = os.environ["GNUPGHOME"] + production = args.target == "production" + try: + fingerprint = sign.signing_fingerprint(homedir, RELEASE_KEY_FINGERPRINT, production) + except sign.SigningError as error: + finish([str(error)], "Signing key") + if production: + shutil.copyfile(PUBLIC_KEY, args.public_key_out) + else: + sign.export_public_key(homedir, fingerprint, args.public_key_out) + signed = sign.sign_staging(args.staging, homedir, os.environ["SIGNING_KEY_PASSPHRASE"], fingerprint) + finish(verify.verify_signatures(args.staging, Path(args.public_key_out)), f"Signing ({len(signed)} files with {fingerprint})") + + +def cmd_upload(args): + """Uploads the signed staging folder create-only.""" + _, plan = read_plan(args.plan) + try: + upload.upload_release(bucket_from_env(), args.staging, plan) + except upload.UploadError as error: + finish([str(error), "this version number is burned; release the next patch version"], "Upload") + finish([], "Upload") + + +def cmd_index(args): + """Rewrites maven-metadata.xml of the plan's artifacts, or of every artifact of this repository with --all.""" + config = load_config() + if args.all: + targets = [(spec["group"], spec["artifact"]) for spec in config["artifacts"].values()] + else: + _, plan = read_plan(args.plan) + targets = [(artifact.group, artifact.artifact) for artifact in plan.artifacts] + bucket = bucket_from_env() + now = datetime.datetime.now(datetime.timezone.utc) + for group, name in targets: + index.rewrite_index(bucket, group, name, now) + finish([], "Version index") + + +def cmd_verify_public(args): + """Downloads the release from the public address and checks files, index and signatures.""" + config, plan = read_plan(args.plan) + data = json.loads(Path(args.manifest).read_text(encoding="utf-8")) + problems = verify.verify_public(data, plan, config["publicBaseUrl"][args.target], args.nocache, args.work) + if not problems: + problems = verify.verify_signatures(args.work, Path(args.public_key)) + finish(problems, "Public files") + + +def cmd_consumer(args): + """Builds a clean Gradle project and a clean Maven project against every artifact of the staging folder, used as a + local repository, and runs the Java 8 artifacts on a Java 8 runtime, so a release that a clean project cannot build + with stops before the approval.""" + _, plan = read_plan(args.plan) + repository = Path(args.staging).resolve().as_uri() + gradlew = str(REPO_ROOT / ("gradlew.bat" if os.name == "nt" else "gradlew")) + problems = [] + lines = [] + for artifact in plan.artifacts: + name = f"{artifact.coordinates}:{plan.version}" + if subprocess.run(consumers.gradle_command(gradlew, repository, artifact, plan.version), cwd=REPO_ROOT).returncode != 0: + problems.append(f"a clean Gradle project could not build with {name}") + elif artifact.consumer_java == 8: + smoke = subprocess.run(consumers.smoke_command(args.java8_home, REPO_ROOT), cwd=REPO_ROOT, capture_output=True, text=True) + print(smoke.stdout + smoke.stderr) + if smoke.returncode != 0: + problems.append(f"{name} did not load on the Java 8 runtime") + else: + lines.append(f"- Java 8 run of {artifact.coordinates}: {smoke.stdout.strip()}") + local_repository = Path(args.work) / f"m2-{artifact.artifact}" + shutil.rmtree(local_repository, ignore_errors=True) + if subprocess.run(consumers.maven_command(args.mvn, repository, artifact, plan.version, local_repository.resolve()), cwd=REPO_ROOT).returncode != 0: + problems.append(f"a clean Maven project could not build with {name}") + else: + problems += consumers.maven_source_problems(local_repository, plan, artifact) + summary(lines) + finish(problems, "Consumer builds") + + +def main(argv=None): + """Parses the subcommand and runs it.""" + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + + command = commands.add_parser("plan") + command.add_argument("--tag", required=True) + command.add_argument("--target", choices=TARGETS, required=True) + command.add_argument("--prerelease", default="unknown") + command.add_argument("--commit", default="") + command.add_argument("--out", required=True) + command.set_defaults(func=cmd_plan) + + command = commands.add_parser("checkout-tags") + command.add_argument("--branch", required=True) + command.set_defaults(func=cmd_checkout_tags) + + command = commands.add_parser("check-branch") + command.add_argument("--plan", required=True) + command.add_argument("--commit", required=True) + command.set_defaults(func=cmd_check_branch) + + command = commands.add_parser("check-sources") + command.add_argument("--plan", required=True) + command.set_defaults(func=cmd_check_sources) + + command = commands.add_parser("check-absent") + command.add_argument("--plan", required=True) + command.add_argument("--target", choices=TARGETS, required=True) + command.add_argument("--nocache", required=True) + command.set_defaults(func=cmd_check_absent) + + command = commands.add_parser("check-required") + command.add_argument("--check-runs", required=True) + command.set_defaults(func=cmd_check_required) + + command = commands.add_parser("stage") + command.add_argument("--plan", required=True) + command.add_argument("--staging", required=True) + command.add_argument("--commit", required=True) + command.add_argument("--target", choices=TARGETS, required=True) + command.add_argument("--manifest", required=True) + command.set_defaults(func=cmd_stage) + + command = commands.add_parser("validate-sbom") + command.add_argument("--plan", required=True) + command.add_argument("--staging", required=True) + command.add_argument("--cli", required=True) + command.set_defaults(func=cmd_validate_sbom) + + command = commands.add_parser("contract") + command.add_argument("--plan", required=True) + command.add_argument("--staging", required=True) + command.add_argument("--capture", action="store_true") + command.set_defaults(func=cmd_contract) + + command = commands.add_parser("verify-manifest") + command.add_argument("--manifest", required=True) + command.add_argument("--staging", required=True) + command.set_defaults(func=cmd_verify_manifest) + + command = commands.add_parser("sign") + command.add_argument("--staging", required=True) + command.add_argument("--target", choices=TARGETS, required=True) + command.add_argument("--public-key-out", required=True) + command.set_defaults(func=cmd_sign) + + command = commands.add_parser("upload") + command.add_argument("--plan", required=True) + command.add_argument("--staging", required=True) + command.set_defaults(func=cmd_upload) + + command = commands.add_parser("index") + command.add_argument("--plan") + command.add_argument("--all", action="store_true") + command.set_defaults(func=cmd_index) + + command = commands.add_parser("verify-public") + command.add_argument("--plan", required=True) + command.add_argument("--manifest", required=True) + command.add_argument("--target", choices=TARGETS, required=True) + command.add_argument("--nocache", required=True) + command.add_argument("--work", required=True) + command.add_argument("--public-key", default=str(PUBLIC_KEY)) + command.set_defaults(func=cmd_verify_public) + + command = commands.add_parser("consumer") + command.add_argument("--plan", required=True) + command.add_argument("--staging", required=True) + command.add_argument("--java8-home", required=True) + command.add_argument("--mvn", default="mvn") + command.add_argument("--work", required=True) + command.set_defaults(func=cmd_consumer) + + args = parser.parse_args(argv) + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/.github/release/s3.py b/.github/release/s3.py new file mode 100644 index 000000000..064fe2fa6 --- /dev/null +++ b/.github/release/s3.py @@ -0,0 +1,54 @@ +"""Minimal R2 access through the AWS CLI (S3 API), so the release job needs no Python packages.""" + +import json +import os +import subprocess + + +class S3Error(Exception): + """An S3 call failed for a reason other than 'not found' or 'already exists'.""" + + +class R2Bucket: + """One R2 bucket, reached with `aws s3api` at the account's R2 endpoint using the AWS_* credentials in the environment.""" + + def __init__(self, bucket, account_id, runner=subprocess.run): + self.bucket = bucket + self.endpoint = f"https://{account_id}.r2.cloudflarestorage.com" + self.runner = runner + + def _aws(self, *args): + env = dict(os.environ, AWS_DEFAULT_REGION="auto", AWS_REQUEST_CHECKSUM_CALCULATION="when_required", AWS_RESPONSE_CHECKSUM_VALIDATION="when_required") + command = ["aws", "s3api", *args, "--bucket", self.bucket, "--endpoint-url", self.endpoint, "--output", "json"] + return self.runner(command, capture_output=True, text=True, env=env) + + def head(self, key): + """Returns {'etag': MD5 hex, 'size': bytes} or None when the key does not exist.""" + result = self._aws("head-object", "--key", key) + if result.returncode == 0: + data = json.loads(result.stdout) + return {"etag": data["ETag"].strip('"'), "size": int(data["ContentLength"])} + if "Not Found" in result.stderr or "(404)" in result.stderr: + return None + raise S3Error(result.stderr.strip()) + + def put_file(self, key, path, content_type, cache_control, create_only): + """Uploads a file in one request; with create_only an existing key is never overwritten and 'exists' is returned.""" + args = ["put-object", "--key", key, "--body", str(path), "--content-type", content_type, "--cache-control", cache_control] + if create_only: + args += ["--if-none-match", "*"] + result = self._aws(*args) + if result.returncode == 0: + return "created" + if create_only and ("PreconditionFailed" in result.stderr or "(412)" in result.stderr): + return "exists" + raise S3Error(result.stderr.strip()) + + def list_keys(self, prefix): + """Every key under the prefix (the CLI follows continuation tokens itself).""" + result = self._aws("list-objects-v2", "--prefix", prefix) + if result.returncode != 0: + raise S3Error(result.stderr.strip()) + if not result.stdout.strip(): + return [] + return [item["Key"] for item in json.loads(result.stdout).get("Contents") or []] diff --git a/.github/release/sbom.py b/.github/release/sbom.py new file mode 100644 index 000000000..9c71e3618 --- /dev/null +++ b/.github/release/sbom.py @@ -0,0 +1,64 @@ +"""Turns the CycloneDX plugin's per-project SBOM into the published component list of one artifact.""" + +import json +from pathlib import Path +from urllib.parse import parse_qs, quote, urlsplit + +from layout import write_checksums + + +def purl(artifact, version, public_base_url): + """Package URL of a published Countly artifact, naming the Countly repository.""" + repository = quote(public_base_url.rstrip("/"), safe="") + return f"pkg:maven/{artifact.group}/{artifact.artifact}@{version}?repository_url={repository}&type=jar" + + +class SbomError(Exception): + """The plugin's SBOM names a module of this build that is not published.""" + + +def _project_path(component): + """Gradle project path of a component the plugin wrote for a module of this build, or None.""" + return parse_qs(urlsplit(component.get("purl", "")).query).get("project_path", [None])[0] + + +def normalize(bom, artifact, version, timestamp, public_base_url, modules=None, platform_groups=()): + """Sets the published coordinates, the licence and a fixed timestamp, and drops the random serial number. Components + the plugin wrote for other modules of this build (java-ui depends on java) get the coordinates those modules are + published under in the same release; `modules` maps a Gradle project path to its published artifact. Components of + `platform_groups` lose their hashes: those groups publish one file per operating system (JavaFX), so the hashes of + the file the build machine resolved match no file the named package stands for.""" + metadata = bom.setdefault("metadata", {}) + old_ref = metadata.get("component", {}).get("bom-ref") + ref = purl(artifact, version, public_base_url) + metadata["component"] = {"type": "library", "bom-ref": ref, "group": artifact.group, "name": artifact.artifact, "version": version, "purl": ref, "licenses": [{"license": {"id": "MIT"}}]} + metadata["timestamp"] = timestamp + bom.pop("serialNumber", None) + renamed = {} if old_ref is None else {old_ref: ref} + for component in bom.get("components", []): + if component.get("group") in platform_groups: + component.pop("hashes", None) + path = _project_path(component) + if path is None: + continue + published = (modules or {}).get(path) + if published is None: + raise SbomError(f"the SBOM of {artifact.coordinates} names the module {path}, which is not published") + new_ref = purl(published, version, public_base_url) + renamed[component.get("bom-ref")] = new_ref + component.update({"bom-ref": new_ref, "group": published.group, "name": published.artifact, "version": version, "purl": new_ref}) + for dependency in bom.get("dependencies", []): + dependency["ref"] = renamed.get(dependency.get("ref"), dependency.get("ref")) + if "dependsOn" in dependency: + dependency["dependsOn"] = [renamed.get(entry, entry) for entry in dependency["dependsOn"]] + return bom + + +def place_sbom(bom_path, staging_dir, artifact, version, timestamp, public_base_url, modules=None, platform_groups=()): + """Writes the published SBOM and its checksums into the staging folder and returns its path.""" + bom = normalize(json.loads(Path(bom_path).read_text(encoding="utf-8")), artifact, version, timestamp, public_base_url, modules, platform_groups) + target = Path(staging_dir) / artifact.folder(version) / f"{artifact.base_name(version)}-cyclonedx.json" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes((json.dumps(bom, indent=2) + "\n").encode("utf-8")) + write_checksums(target) + return target diff --git a/.github/release/sign.py b/.github/release/sign.py new file mode 100644 index 000000000..822a7dec2 --- /dev/null +++ b/.github/release/sign.py @@ -0,0 +1,55 @@ +"""Signs the staged files with the key of the target: the release key for production, the dry-run key for tests.""" + +import subprocess +from pathlib import Path + +from layout import is_checksum + + +class SigningError(Exception): + """The keyring holds the wrong signing key for the target.""" + + +def primary_secret_fingerprints(homedir, runner=subprocess.run): + """Fingerprints of the primary secret keys in the keyring, without their subkeys.""" + listing = runner(["gpg", "--homedir", str(homedir), "--batch", "--with-colons", "--list-secret-keys"], capture_output=True, text=True, check=True).stdout + lines = listing.splitlines() + return [line.split(":")[9] for index, line in enumerate(lines) if line.startswith("fpr:") and index > 0 and lines[index - 1].startswith("sec:")] + + +def signing_fingerprint(homedir, release_fingerprint, production, runner=subprocess.run): + """The key to sign with. Production requires the release key. A dry run requires exactly one other key and + refuses the release key, so the release key never has to leave the production environment.""" + fingerprints = primary_secret_fingerprints(homedir, runner) + if production: + if release_fingerprint not in fingerprints: + raise SigningError(f"the imported signing key is not {release_fingerprint}") + return release_fingerprint + if release_fingerprint in fingerprints: + raise SigningError("a dry run must not use the release key; the maven-test environment needs its own dry-run key") + if len(fingerprints) != 1: + raise SigningError(f"a dry run needs exactly one signing key in the keyring, found {len(fingerprints)}") + return fingerprints[0] + + +def export_public_key(homedir, fingerprint, path, runner=subprocess.run): + """Writes the ASCII-armoured public key of the fingerprint to path.""" + armored = runner(["gpg", "--homedir", str(homedir), "--batch", "--armor", "--export", fingerprint], capture_output=True, text=True, check=True).stdout + Path(path).write_bytes(armored.encode("ascii")) + + +def files_to_sign(staging_dir): + """Every staged file except checksum files and existing signatures.""" + return sorted(path for path in Path(staging_dir).rglob("*") if path.is_file() and not is_checksum(path.name) and not path.name.endswith(".asc")) + + +def sign_staging(staging_dir, homedir, passphrase, fingerprint, runner=subprocess.run): + """Writes one ASCII-armoured detached signature, made by exactly the given key, next to every file to sign; returns the signed files.""" + signed = files_to_sign(staging_dir) + for path in signed: + runner( + ["gpg", "--homedir", str(homedir), "--batch", "--yes", "--pinentry-mode", "loopback", "--passphrase-fd", "0", + "--local-user", f"{fingerprint}!", "--armor", "--detach-sign", "--output", f"{path}.asc", str(path)], + input=passphrase, text=True, capture_output=True, check=True, + ) + return signed diff --git a/.github/release/sources.py b/.github/release/sources.py new file mode 100644 index 000000000..664d8180b --- /dev/null +++ b/.github/release/sources.py @@ -0,0 +1,28 @@ +"""Checks that every version source of the tag's artifacts carries the tag's version.""" + +import re +from pathlib import Path + + +def check_version_sources(repo_root, plan, config): + """Returns problems; empty when every version source and, for final releases, every changelog heading matches.""" + problems = [] + root = Path(repo_root) + for artifact in plan.artifacts: + spec = config["artifacts"][artifact.key] + for source in spec["versionSources"]: + path = root / source["file"] + if not path.is_file(): + problems.append(f"{source['file']} does not exist") + continue + match = re.search(source["regex"], path.read_text(encoding="utf-8"), re.MULTILINE) + if match is None: + problems.append(f"{source['file']}: no version found") + elif match.group(1).strip() != plan.version: + problems.append(f"{source['file']}: version is {match.group(1).strip()}, the tag says {plan.version}") + changelog = spec.get("changelog") + if changelog and not plan.prerelease: + text = (root / changelog).read_text(encoding="utf-8") + if re.search(rf"^## {re.escape(plan.version)}\s*$", text, re.MULTILINE) is None: + problems.append(f"{changelog}: no '## {plan.version}' heading") + return list(dict.fromkeys(problems)) diff --git a/.github/release/tests/__init__.py b/.github/release/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/.github/release/tests/builders.py b/.github/release/tests/builders.py new file mode 100644 index 000000000..bf6b5dc25 --- /dev/null +++ b/.github/release/tests/builders.py @@ -0,0 +1,59 @@ +"""Builders for staged artifacts: class files, jars, POMs and Gradle module files.""" + +import io +import json +import zipfile +from pathlib import Path + +from layout import primary_files, write_checksums + + +def class_bytes(major): + """A minimal class-file header with the given major version.""" + return b"\xca\xfe\xba\xbe" + (0).to_bytes(2, "big") + major.to_bytes(2, "big") + b"\x00" * 8 + + +def jar_bytes(majors): + """A jar holding one class file per major version.""" + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + for number, major in enumerate(majors): + archive.writestr(f"ly/count/C{number}.class", class_bytes(major)) + return buffer.getvalue() + + +def pom_bytes(dependencies): + """A POM with the given group:artifact:version:scope dependencies.""" + entries = "".join( + f"{g}{a}{v}{s}" + for g, a, v, s in (dependency.split(":") for dependency in dependencies) + ) + return f'4.0.0{entries}'.encode("utf-8") + + +def module_bytes(jvm_version=None, dependencies=()): + """A Gradle module file with a runtime library variant (optionally with org.gradle.jvm.version and group:module:version dependencies) and a sources variant.""" + attributes = {"org.gradle.category": "library", "org.gradle.usage": "java-runtime"} + if jvm_version is not None: + attributes["org.gradle.jvm.version"] = jvm_version + declared = [] + for dependency in dependencies: + group, module, version = dependency.split(":") + declared.append({"group": group, "module": module, "version": {"requires": version}}) + variants = [{"name": "runtime", "attributes": attributes, "dependencies": declared}, {"name": "sources", "attributes": {"org.gradle.category": "documentation"}}] + return json.dumps({"formatVersion": "1.1", "variants": variants}).encode("utf-8") + + +def write_artifact(staging, artifact, version, main=None, pom=None, module=None): + """Writes a complete staged artifact version (primary files plus four checksums each) and returns its folder.""" + folder = Path(staging) / artifact.folder(version) + folder.mkdir(parents=True, exist_ok=True) + base = artifact.base_name(version) + contents = {name: f"content of {name}".encode("utf-8") for name in primary_files(artifact, version)} + contents[f"{base}.jar"] = main if main is not None else jar_bytes([52]) + contents[f"{base}.module"] = module if module is not None else module_bytes() + contents[f"{base}.pom"] = pom if pom is not None else pom_bytes([]) + for name, data in contents.items(): + (folder / name).write_bytes(data) + write_checksums(folder / name) + return folder diff --git a/.github/release/tests/support.py b/.github/release/tests/support.py new file mode 100644 index 000000000..374169c3a --- /dev/null +++ b/.github/release/tests/support.py @@ -0,0 +1,53 @@ +"""Shared test helpers: the repository configuration and in-memory stand-ins for R2 and the web.""" + +import hashlib +from pathlib import Path + +from plan import load_config + + +def repository_config(): + """The real config.json of this repository.""" + return load_config(Path(__file__).resolve().parents[1] / "config.json") + + +class FakeBucket: + """In-memory stand-in for s3.R2Bucket with MD5 ETags; `fail_on` makes one upload fail like a lost connection.""" + + def __init__(self, fail_on=None): + self.objects = {} + self.calls = [] + self.fail_on = fail_on + + def head(self, key): + """Returns {'etag', 'size'} or None.""" + if key not in self.objects: + return None + data = self.objects[key][0] + return {"etag": hashlib.md5(data).hexdigest(), "size": len(data)} + + def put_file(self, key, path, content_type, cache_control, create_only): + """Stores a file; with create_only an existing key is left alone and 'exists' is returned.""" + if key == self.fail_on: + raise RuntimeError(f"connection lost while uploading {key}") + self.calls.append(key) + if create_only and key in self.objects: + return "exists" + self.objects[key] = (Path(path).read_bytes(), content_type, cache_control) + return "created" + + def list_keys(self, prefix): + """Keys starting with the prefix, sorted.""" + return sorted(key for key in self.objects if key.startswith(prefix)) + + +class FakeWeb: + """Serves a dict of URL to bytes (ignoring ?nocache=...); anything else is a 404 (None).""" + + def __init__(self, pages=None): + self.pages = dict(pages or {}) + self.requests = [] + + def __call__(self, url): + self.requests.append(url) + return self.pages.get(url.split("?nocache=")[0]) diff --git a/.github/release/tests/test_checks.py b/.github/release/tests/test_checks.py new file mode 100644 index 000000000..d71029585 --- /dev/null +++ b/.github/release/tests/test_checks.py @@ -0,0 +1,43 @@ +import subprocess +import unittest + +from checks import branch_problems, missing_checks + +COMMIT = "c" * 40 + + +class ChecksTest(unittest.TestCase): + def test_reports_missing_and_failed_checks(self): + runs = {"check_runs": [{"name": "Build and Test", "conclusion": "success"}, {"name": "Analyze (java)", "conclusion": "failure"}]} + required = ["Build and Test", "Analyze (java)", "OSV scan (all modules)"] + self.assertEqual(missing_checks(runs, required), ["Analyze (java)", "OSV scan (all modules)"]) + + def test_every_run_of_a_required_check_must_succeed(self): + runs = {"check_runs": [ + {"name": "Build and Test", "conclusion": "failure"}, + {"name": "Build and Test", "conclusion": "success"}, + {"name": "Analyze (java)", "conclusion": "success"}, + {"name": "Analyze (java)", "conclusion": None}, + {"name": "OSV scan (all modules)", "conclusion": "success"}, + {"name": "OSV scan (all modules)", "conclusion": "success"}, + ]} + required = ["Build and Test", "Analyze (java)", "OSV scan (all modules)"] + self.assertEqual(missing_checks(runs, required), ["Build and Test", "Analyze (java)"]) + + def test_tag_commit_must_be_on_the_release_branch(self): + calls = [] + + def on_branch(args, **kwargs): + calls.append(args) + return subprocess.CompletedProcess(args, 0) + + def elsewhere(args, **kwargs): + return subprocess.CompletedProcess(args, 1) + + self.assertEqual(branch_problems(COMMIT, "staging", "/repo", runner=on_branch), []) + self.assertEqual(calls[0], ["git", "-C", "/repo", "merge-base", "--is-ancestor", COMMIT, "origin/staging"]) + self.assertEqual(branch_problems(COMMIT, "master", "/repo", runner=elsewhere), [f"{COMMIT} is not on origin/master; the tag must point to that branch"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/tests/test_consumers.py b/.github/release/tests/test_consumers.py new file mode 100644 index 000000000..09e6f274d --- /dev/null +++ b/.github/release/tests/test_consumers.py @@ -0,0 +1,50 @@ +import tempfile +import unittest +from pathlib import Path + +from consumers import gradle_command, maven_command, maven_source_problems, smoke_command +from plan import plan_for_tag +from tests.support import repository_config + + +class ConsumersTest(unittest.TestCase): + def setUp(self): + self.plan = plan_for_tag(repository_config(), "26.8.1") + self.java, self.ui = self.plan.artifacts + + def test_gradle_builds_each_artifact_at_its_java_level(self): + command = gradle_command("/repo/gradlew", "file:///staging", self.ui, "26.8.1") + self.assertEqual(command[:3], ["/repo/gradlew", "-p", ".github/release/consumers/gradle"]) + for part in ["installDist", "-PcountlyRepository=file:///staging", "-PcountlyDependency=ly.count.sdk:java-ui:26.8.1", "-PcountlyRelease=17", "-PcountlyProbe=java-ui"]: + self.assertIn(part, command) + self.assertIn("-PcountlyRelease=8", gradle_command("/repo/gradlew", "file:///staging", self.java, "26.8.1")) + + def test_maven_uses_an_empty_local_repository(self): + command = maven_command("mvn", "file:///staging", self.java, "26.8.1", "/work/m2-java") + self.assertEqual(command[:4], ["mvn", "-B", "-f", ".github/release/consumers/maven/pom.xml"]) + for part in ["-Dcountly.artifact=java", "-Dcountly.version=26.8.1", "-Dcountly.release=8", "-Dmaven.repo.local=/work/m2-java"]: + self.assertIn(part, command) + + def test_smoke_run_uses_the_given_runtime_and_the_installed_classpath(self): + command = smoke_command("/jdk8", "/repo") + self.assertEqual(Path(command[0]), Path("/jdk8/bin/java")) + self.assertEqual(Path(command[2]), Path("/repo/.github/release/consumers/gradle/build/install/countly-consumer/lib/*")) + self.assertEqual(command[3], "ly.count.consumer.Probe") + + def write_record(self, local, artifact, text): + """Writes Maven's download record for one artifact version into the local repository.""" + folder = Path(local) / artifact.folder("26.8.1") + folder.mkdir(parents=True, exist_ok=True) + (folder / "_remote.repositories").write_text(text, encoding="utf-8") + + def test_maven_must_resolve_from_the_staged_release(self): + local = tempfile.mkdtemp() + self.assertEqual(maven_source_problems(local, self.plan, self.java), ["Maven did not download ly.count.sdk:java:26.8.1"]) + self.write_record(local, self.java, "#NOTE: This is a Maven Resolver internal implementation file\njava-26.8.1.jar>countly=\njava-26.8.1.pom>countly=\n") + self.assertEqual(maven_source_problems(local, self.plan, self.java), []) + self.write_record(local, self.ui, "java-ui-26.8.1.jar>central=\njava-ui-26.8.1.pom>countly=\n") + self.assertEqual(maven_source_problems(local, self.plan, self.ui), ["Maven resolved ly.count.sdk:java-ui:26.8.1 from another repository than the staged release"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/tests/test_contract.py b/.github/release/tests/test_contract.py new file mode 100644 index 000000000..e984c6fda --- /dev/null +++ b/.github/release/tests/test_contract.py @@ -0,0 +1,69 @@ +import tempfile +import unittest +from pathlib import Path + +from contract import actual_contract, compare, max_class_major, module_dependency_problems, summary_rows +from plan import plan_for_tag +from tests.builders import jar_bytes, module_bytes, pom_bytes, write_artifact +from tests.support import repository_config + +JSON = "org.json:json:20250517" +JSR305 = "com.google.code.findbugs:jsr305:3.0.2" +JAVA_CONTRACT = { + "schema": 1, + "coordinates": "ly.count.sdk:java", + "pomDependencies": [JSR305 + ":runtime", JSON + ":runtime"], + "moduleVariantAttributes": {"org.gradle.jvm.version": 8}, + "maxClassFileMajor": 52, +} + + +class ContractTest(unittest.TestCase): + def setUp(self): + self.config = repository_config() + self.staging = Path(tempfile.mkdtemp()) + self.java = plan_for_tag(self.config, "26.8.1").artifacts[0] + + def stage_java(self, majors=(52,), jvm_version=8, pom_dependencies=(JSON + ":runtime", JSR305 + ":runtime"), module_dependencies=(JSON, JSR305)): + """Stages java 26.8.1 with the given contents and returns the contract it actually fulfils.""" + write_artifact(self.staging, self.java, "26.8.1", main=jar_bytes(list(majors)), pom=pom_bytes(list(pom_dependencies)), module=module_bytes(jvm_version, module_dependencies)) + return actual_contract(self.staging, self.java, "26.8.1") + + def test_actual_contract_has_the_contract_file_shape(self): + self.assertEqual(self.stage_java(), JAVA_CONTRACT) + + def test_matching_contract(self): + self.assertEqual(compare(JAVA_CONTRACT, self.stage_java()), []) + self.assertEqual(module_dependency_problems(self.staging, self.java, "26.8.1"), []) + + def test_regressions_are_reported(self): + actual = self.stage_java(majors=(52, 61), jvm_version=17, pom_dependencies=[JSON + ":compile"]) + problems = compare(JAVA_CONTRACT, actual) + joined = "\n".join(problems) + self.assertEqual(len(problems), 3, joined) + for fragment in ["POM dependencies", "org.gradle.jvm.version is 17", "major version 61"]: + self.assertIn(fragment, joined) + + def test_dependency_only_in_the_module_file_is_reported(self): + self.stage_java(module_dependencies=(JSON, JSR305, "org.slf4j:slf4j-api:2.0.17")) + self.assertEqual(module_dependency_problems(self.staging, self.java, "26.8.1"), [ + f"the .module declares ['{JSR305}', '{JSON}', 'org.slf4j:slf4j-api:2.0.17'] but the POM declares ['{JSR305}', '{JSON}']", + ]) + + def test_own_release_version_is_a_placeholder(self): + ui = plan_for_tag(self.config, "26.8.1").artifacts[1] + pom = pom_bytes(["ly.count.sdk:java:26.8.1:compile", "ly.count.sdk:java:26.8.0:compile", JSON + ":runtime"]) + write_artifact(self.staging, ui, "26.8.1", pom=pom, module=module_bytes(17, ["ly.count.sdk:java:26.8.1", JSON])) + self.assertEqual(actual_contract(self.staging, ui, "26.8.1")["pomDependencies"], ["ly.count.sdk:java:26.8.0:compile", "ly.count.sdk:java:{version}:compile", JSON + ":runtime"]) + + def test_highest_class_version_wins(self): + self.assertEqual(max_class_major(jar_bytes([52, 50])), 52) + + def test_summary_rows(self): + rows = summary_rows(JAVA_CONTRACT, self.stage_java()) + self.assertIn(("org.gradle.jvm.version", "8", "8"), rows) + self.assertIn(("highest class file version", "at most 52", "52"), rows) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/tests/test_index.py b/.github/release/tests/test_index.py new file mode 100644 index 000000000..a0540928b --- /dev/null +++ b/.github/release/tests/test_index.py @@ -0,0 +1,69 @@ +import datetime +import hashlib +import unittest + +from index import METADATA_CACHE, complete_versions, listed_versions, render, rewrite_index +from tests.support import FakeBucket +from upload import IMMUTABLE + +NOW = datetime.datetime(2026, 10, 27, 10, 15, 0) + + +def quiet(line): + """Swallows log lines.""" + + +class IndexTest(unittest.TestCase): + def test_complete_versions_need_a_pom(self): + keys = [ + "ly/count/sdk/java/26.8.1/java-26.8.1.pom", + "ly/count/sdk/java/26.8.1/java-26.8.1.jar", + "ly/count/sdk/java/26.9.0/java-26.9.0.jar", + "ly/count/sdk/java/maven-metadata.xml", + "ly/count/sdk/java-ui/26.8.1/java-ui-26.8.1.pom", + ] + self.assertEqual(complete_versions(keys, "ly/count/sdk", "java"), {"26.8.1"}) + + def test_only_final_versions_are_listed_in_numeric_order(self): + folders = {"26.1.10", "26.2.0-rc1", "26.1.6-rc2", "26.1.6", "26.10.0", "26.9.1", "19.09-sdk2-rc", "21.11.0-RC1", "26.3.0-hotfix"} + self.assertEqual(listed_versions(folders), ["26.1.6", "26.1.10", "26.9.1", "26.10.0"]) + + def test_render(self): + self.assertEqual(render("ly.count.sdk", "java", ["26.1.10", "26.2.0"], NOW), "\n".join([ + '', + "", + " ly.count.sdk", + " java", + " ", + " 26.2.0", + " 26.2.0", + " ", + " 26.1.10", + " 26.2.0", + " ", + " 20261027101500", + " ", + "", + "", + ])) + + def test_rewrite_writes_checksums_first_and_the_body_last(self): + bucket = FakeBucket() + for version in ["26.1.6", "26.2.0", "26.2.0-rc1"]: + bucket.objects[f"ly/count/sdk/java/{version}/java-{version}.pom"] = (b"pom", "application/xml", IMMUTABLE) + self.assertEqual(rewrite_index(bucket, "ly.count.sdk", "java", NOW, log=quiet), ["26.1.6", "26.2.0"]) + self.assertEqual(bucket.calls, [f"ly/count/sdk/java/maven-metadata.xml.{suffix}" for suffix in ["md5", "sha1", "sha256", "sha512"]] + ["ly/count/sdk/java/maven-metadata.xml"]) + body, kind, cache = bucket.objects["ly/count/sdk/java/maven-metadata.xml"] + self.assertEqual((kind, cache), ("application/xml", METADATA_CACHE)) + self.assertEqual(bucket.objects["ly/count/sdk/java/maven-metadata.xml.sha1"][0], hashlib.sha1(body).hexdigest().encode("ascii")) + self.assertIn(b"26.2.0", body) + + def test_no_listed_version_writes_nothing(self): + bucket = FakeBucket() + bucket.objects["ly/count/sdk/java/26.2.0-rc1/java-26.2.0-rc1.pom"] = (b"pom", "application/xml", IMMUTABLE) + self.assertEqual(rewrite_index(bucket, "ly.count.sdk", "java", NOW, log=quiet), []) + self.assertEqual(bucket.calls, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/tests/test_layout.py b/.github/release/tests/test_layout.py new file mode 100644 index 000000000..b66a1c72c --- /dev/null +++ b/.github/release/tests/test_layout.py @@ -0,0 +1,64 @@ +import tempfile +import unittest +from pathlib import Path + +from layout import check_staging, expected_files, primary_files, remove_index_files, write_checksums +from plan import plan_for_tag +from tests.builders import write_artifact +from tests.support import repository_config + + +class LayoutTest(unittest.TestCase): + def setUp(self): + self.config = repository_config() + self.staging = Path(tempfile.mkdtemp()) + self.plan = plan_for_tag(self.config, "26.8.1") + + def test_expected_files_of_a_jar(self): + ui = self.plan.artifacts[1] + self.assertEqual(primary_files(ui, "26.8.1"), ["java-ui-26.8.1.jar", "java-ui-26.8.1.pom", "java-ui-26.8.1.module", "java-ui-26.8.1-sources.jar", "java-ui-26.8.1-javadoc.jar", "java-ui-26.8.1-cyclonedx.json"]) + self.assertEqual(len(expected_files(ui, "26.8.1")), 30) + + def test_checksums_are_plain_hex(self): + path = self.staging / "a.txt" + path.write_bytes(b"abc") + write_checksums(path) + self.assertEqual((self.staging / "a.txt.md5").read_bytes(), b"900150983cd24fb0d6963f7d28e17f72") + self.assertEqual((self.staging / "a.txt.sha1").read_bytes(), b"a9993e364706816aba3e25717850c26c9cd0d89d") + + def test_complete_staging_passes(self): + for artifact in self.plan.artifacts: + write_artifact(self.staging, artifact, self.plan.version) + self.assertEqual(check_staging(self.staging, self.plan), []) + + def test_both_artifacts_are_required(self): + write_artifact(self.staging, self.plan.artifacts[0], self.plan.version) + problems = check_staging(self.staging, self.plan) + self.assertEqual(len(problems), 30) + self.assertIn("missing ly/count/sdk/java-ui/26.8.1/java-ui-26.8.1.jar", problems) + + def test_missing_unexpected_and_wrong_files_are_reported(self): + folder = write_artifact(self.staging, self.plan.artifacts[0], self.plan.version) + write_artifact(self.staging, self.plan.artifacts[1], self.plan.version) + (folder / "java-26.8.1-javadoc.jar").unlink() + (folder / "notes.txt").write_bytes(b"x") + (folder / "java-26.8.1.pom.sha1").write_bytes(b"0" * 40) + (self.staging / "ly/count/sdk/other").mkdir(parents=True) + (self.staging / "ly/count/sdk/other/x.jar").write_bytes(b"x") + self.assertEqual(check_staging(self.staging, self.plan), [ + "missing ly/count/sdk/java/26.8.1/java-26.8.1-javadoc.jar", + "unexpected ly/count/sdk/java/26.8.1/notes.txt", + "unexpected ly/count/sdk/other/x.jar", + "wrong checksum ly/count/sdk/java/26.8.1/java-26.8.1.pom.sha1", + ]) + + def test_index_files_are_removed(self): + path = self.staging / "ly/count/sdk/java/maven-metadata.xml.sha1" + path.parent.mkdir(parents=True) + path.write_bytes(b"x") + remove_index_files(self.staging) + self.assertFalse(path.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/tests/test_manifest.py b/.github/release/tests/test_manifest.py new file mode 100644 index 000000000..144786170 --- /dev/null +++ b/.github/release/tests/test_manifest.py @@ -0,0 +1,23 @@ +import hashlib +import tempfile +import unittest +from pathlib import Path + +from manifest import build_manifest, verify_manifest + + +class ManifestTest(unittest.TestCase): + def test_round_trip_and_changes(self): + staging = Path(tempfile.mkdtemp()) + (staging / "a").mkdir() + (staging / "a/x.pom").write_bytes(b"one") + manifest = build_manifest(staging, "26.2.0", "c" * 40) + self.assertEqual(manifest, {"tag": "26.2.0", "commit": "c" * 40, "files": [{"path": "a/x.pom", "size": 3, "sha256": hashlib.sha256(b"one").hexdigest()}]}) + self.assertEqual(verify_manifest(staging, manifest), []) + (staging / "a/x.pom").write_bytes(b"two") + (staging / "a/y.pom").write_bytes(b"new") + self.assertEqual(verify_manifest(staging, manifest), ["unexpected a/y.pom", "changed a/x.pom"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/tests/test_plan.py b/.github/release/tests/test_plan.py new file mode 100644 index 000000000..9fab6c50d --- /dev/null +++ b/.github/release/tests/test_plan.py @@ -0,0 +1,65 @@ +import tempfile +import unittest +from pathlib import Path + +from plan import PlanError, checkout_tags, plan_for_tag, plan_from_json +from tests.support import repository_config + +BOTH = ["ly.count.sdk:java", "ly.count.sdk:java-ui"] + + +class PlanTest(unittest.TestCase): + def setUp(self): + self.config = repository_config() + + def test_tags(self): + cases = [ + ("26.8.1", "26.8.1", True), + ("26.8.1-rc1", "26.8.1-rc1", False), + ("0.0.1", "0.0.1", True), + ] + for tag, version, listed in cases: + with self.subTest(tag=tag): + plan = plan_for_tag(self.config, tag) + self.assertEqual(plan.branch, "staging") + self.assertEqual([artifact.coordinates for artifact in plan.artifacts], BOTH) + self.assertEqual(plan.version, version) + self.assertEqual(plan.listed, listed) + self.assertEqual(plan.prerelease, not listed) + + def test_refused_tags(self): + refused = ["v26.8.1", "26.8", "26.8.1-RC1", "26.8.1-rc.1", " 26.8.1", "26.8.1-nw", "native-26.8.1", "plugin-26.8.1", "26.8.1/../x", "java-26.8.1", "٢٦.8.1"] + for tag in refused: + with self.subTest(tag=tag): + with self.assertRaises(PlanError): + plan_for_tag(self.config, tag) + + def test_modules_and_round_trip(self): + plan = plan_for_tag(self.config, "26.8.1") + self.assertEqual(plan.modules, [":sdk-java", ":sdk-java-ui"]) + self.assertEqual(plan.artifacts[1].module_dir, "sdk-java-ui") + self.assertEqual(plan.artifacts[1].folder("26.8.1"), "ly/count/sdk/java-ui/26.8.1") + self.assertEqual(plan_from_json(self.config, plan.to_json()), plan) + + def test_artifact_details(self): + java, ui = plan_for_tag(self.config, "26.8.1").artifacts + self.assertEqual((java.contract, java.consumer_java), (".github/release-contract/java.json", 8)) + self.assertEqual((ui.contract, ui.consumer_java), (".github/release-contract/java-ui.json", 17)) + + def test_checkout_tags_follow_the_branch(self): + root = Path(tempfile.mkdtemp()) + (root / "gradle.properties").write_bytes(b"VERSION_NAME=26.8.1-rc1\r\nGROUP=ly.count.sdk\r\n") + self.assertEqual(checkout_tags(self.config, root, "staging"), ["26.8.1-rc1"]) + self.assertEqual(checkout_tags(self.config, root, "master"), []) + + def test_checkout_tags_need_a_readable_version(self): + root = Path(tempfile.mkdtemp()) + with self.assertRaises(PlanError): + checkout_tags(self.config, root, "staging") + (root / "gradle.properties").write_bytes(b"GROUP=ly.count.sdk\n") + with self.assertRaises(PlanError): + checkout_tags(self.config, root, "staging") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/tests/test_release_cli.py b/.github/release/tests/test_release_cli.py new file mode 100644 index 000000000..f9cec37e7 --- /dev/null +++ b/.github/release/tests/test_release_cli.py @@ -0,0 +1,65 @@ +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +RELEASE = Path(__file__).resolve().parents[1] / "release.py" + + +def run_cli(*args, output=""): + """Runs release.py like the workflow does; GitHub's step output file is written only when `output` names one.""" + env = dict(os.environ, GITHUB_OUTPUT=output, GITHUB_STEP_SUMMARY="") + return subprocess.run([sys.executable, str(RELEASE), *args], capture_output=True, text=True, env=env) + + +class ReleaseCliTest(unittest.TestCase): + def setUp(self): + self.folder = Path(tempfile.mkdtemp()) + + def test_plan_writes_plan_json_and_the_task_lists(self): + out = self.folder / "plan.json" + output = self.folder / "output" + output.write_bytes(b"") + result = run_cli("plan", "--tag", "26.8.1", "--target", "test", "--out", str(out), output=str(output)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(json.loads(out.read_text(encoding="utf-8"))["artifacts"], ["java", "java-ui"]) + lines = dict(line.split("=", 1) for line in output.read_text(encoding="utf-8").splitlines()) + self.assertEqual(lines["environment"], "maven-test") + self.assertEqual(lines["check_tasks"], ":sdk-java:check :sdk-java-ui:check") + self.assertEqual(lines["published_modules"], ":sdk-java,:sdk-java-ui") + self.assertIn(":sdk-java-ui:publishAllPublicationsToReleaseStagingRepository :sdk-java-ui:cyclonedxDirectBom", lines["publish_tasks"]) + + def test_plan_refuses_an_unknown_tag(self): + result = run_cli("plan", "--tag", "v1", "--target", "test", "--out", str(self.folder / "plan.json")) + self.assertEqual(result.returncode, 1) + self.assertIn("::error::", result.stdout) + + def test_checkout_tags_are_a_json_list_the_matrix_can_read(self): + output = self.folder / "output" + output.write_bytes(b"") + result = run_cli("checkout-tags", "--branch", "staging", output=str(output)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + line = output.read_text(encoding="utf-8").strip() + self.assertTrue(line.startswith("tags="), line) + self.assertEqual(len(json.loads(line[len("tags="):])), 1) + + def test_a_branch_without_releases_gets_the_empty_list_the_workflow_skips_on(self): + output = self.folder / "output" + output.write_bytes(b"") + result = run_cli("checkout-tags", "--branch", "master", output=str(output)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(output.read_text(encoding="utf-8").strip(), "tags=[]") + + def test_prerelease_flag_must_match(self): + cases = [("26.8.1", "true", 1), ("26.8.1-rc1", "false", 1), ("26.8.1", "unknown", 1), ("26.8.1-rc1", "true", 0), ("26.8.1", "false", 0)] + for tag, prerelease, code in cases: + with self.subTest(tag=tag, prerelease=prerelease): + result = run_cli("plan", "--tag", tag, "--target", "production", "--prerelease", prerelease, "--out", str(self.folder / "plan.json")) + self.assertEqual(result.returncode, code, result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/tests/test_s3.py b/.github/release/tests/test_s3.py new file mode 100644 index 000000000..1967aed2d --- /dev/null +++ b/.github/release/tests/test_s3.py @@ -0,0 +1,51 @@ +import subprocess +import unittest + +from s3 import R2Bucket, S3Error + + +class RecordingRunner: + """Stands in for subprocess.run and replays one prepared result.""" + + def __init__(self, returncode=0, stdout="", stderr=""): + self.result = (returncode, stdout, stderr) + self.calls = [] + + def __call__(self, args, **kwargs): + self.calls.append((args, kwargs)) + return subprocess.CompletedProcess(args, *self.result) + + +class R2BucketTest(unittest.TestCase): + def test_create_only_put_uses_if_none_match_and_the_r2_endpoint(self): + runner = RecordingRunner() + bucket = R2Bucket("countly-maven", "abc123", runner=runner) + self.assertEqual(bucket.put_file("a/b.pom", "/tmp/b.pom", "application/xml", "public, max-age=60", create_only=True), "created") + args, kwargs = runner.calls[0] + self.assertEqual(args[:3], ["aws", "s3api", "put-object"]) + self.assertIn("--if-none-match", args) + self.assertEqual(args[args.index("--endpoint-url") + 1], "https://abc123.r2.cloudflarestorage.com") + self.assertEqual(kwargs["env"]["AWS_DEFAULT_REGION"], "auto") + self.assertEqual(kwargs["env"]["AWS_REQUEST_CHECKSUM_CALCULATION"], "when_required") + + def test_existing_key_and_missing_key(self): + exists = R2Bucket("b", "a", runner=RecordingRunner(254, "", "An error occurred (PreconditionFailed) when calling the PutObject operation")) + self.assertEqual(exists.put_file("k", "/tmp/f", "t", "c", create_only=True), "exists") + missing = R2Bucket("b", "a", runner=RecordingRunner(254, "", "An error occurred (404) when calling the HeadObject operation: Not Found")) + self.assertIsNone(missing.head("k")) + + def test_other_errors_raise(self): + broken = R2Bucket("b", "a", runner=RecordingRunner(255, "", "An error occurred (AccessDenied)")) + with self.assertRaises(S3Error): + broken.put_file("k", "/tmp/f", "t", "c", create_only=True) + + def test_head_and_list(self): + head = R2Bucket("b", "a", runner=RecordingRunner(0, '{"ETag": "\\"0cc175b9c0f1b6a831c399e269772661\\"", "ContentLength": 1}')) + self.assertEqual(head.head("k"), {"etag": "0cc175b9c0f1b6a831c399e269772661", "size": 1}) + listing = R2Bucket("b", "a", runner=RecordingRunner(0, '{"Contents": [{"Key": "x/1"}, {"Key": "x/2"}]}')) + self.assertEqual(listing.list_keys("x/"), ["x/1", "x/2"]) + self.assertEqual(R2Bucket("b", "a", runner=RecordingRunner(0, "")).list_keys("x/"), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/tests/test_sbom.py b/.github/release/tests/test_sbom.py new file mode 100644 index 000000000..4d757ef5a --- /dev/null +++ b/.github/release/tests/test_sbom.py @@ -0,0 +1,79 @@ +import copy +import json +import tempfile +import unittest +from pathlib import Path + +from plan import plan_for_tag +from sbom import SbomError, normalize, place_sbom +from tests.support import repository_config + +PLUGIN_BOM = { + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "serialNumber": "urn:uuid:random", + "metadata": { + "timestamp": "2026-10-01T00:00:00Z", + "component": {"type": "application", "bom-ref": "pkg:maven/ly.count.sdk/sdk-java-ui@26.8.1?type=jar", "group": "ly.count.sdk", "name": "sdk-java-ui", "version": "26.8.1"}, + "tools": {"components": [{"type": "application", "name": "cyclonedx-gradle-plugin"}]}, + }, + "components": [{"type": "library", "bom-ref": "pkg:maven/org.json/json@20250517?type=jar", "name": "json"}], + "dependencies": [{"ref": "pkg:maven/ly.count.sdk/sdk-java-ui@26.8.1?type=jar", "dependsOn": ["pkg:maven/org.json/json@20250517?type=jar"]}], +} + + +class SbomTest(unittest.TestCase): + def setUp(self): + self.config = repository_config() + self.java, self.ui = plan_for_tag(self.config, "26.8.1").artifacts + + def test_normalize_sets_the_published_coordinates(self): + bom = normalize(copy.deepcopy(PLUGIN_BOM), self.ui, "26.8.1", "2026-10-27T10:15:00+03:00", "https://maven.countly.com/") + ref = "pkg:maven/ly.count.sdk/java-ui@26.8.1?repository_url=https%3A%2F%2Fmaven.countly.com&type=jar" + self.assertEqual(bom["metadata"]["component"], {"type": "library", "bom-ref": ref, "group": "ly.count.sdk", "name": "java-ui", "version": "26.8.1", "purl": ref, "licenses": [{"license": {"id": "MIT"}}]}) + self.assertEqual(bom["metadata"]["timestamp"], "2026-10-27T10:15:00+03:00") + self.assertNotIn("serialNumber", bom) + self.assertEqual(bom["dependencies"][0]["ref"], ref) + + def test_other_modules_of_the_build_get_their_published_coordinates(self): + core = "pkg:maven/countly-sdk-java/sdk-java@unspecified?project_path=%3Asdk-java" + bom = copy.deepcopy(PLUGIN_BOM) + bom["components"].append({"type": "library", "bom-ref": core, "group": "countly-sdk-java", "name": "sdk-java", "version": "unspecified", "purl": core}) + bom["dependencies"][0]["dependsOn"].append(core) + bom["dependencies"].append({"ref": core, "dependsOn": ["pkg:maven/org.json/json@20250517?type=jar"]}) + normalize(bom, self.ui, "26.8.1", "t", "https://maven.countly.com/", {":sdk-java": self.java, ":sdk-java-ui": self.ui}) + java_ref = "pkg:maven/ly.count.sdk/java@26.8.1?repository_url=https%3A%2F%2Fmaven.countly.com&type=jar" + self.assertEqual(bom["components"][-1], {"type": "library", "bom-ref": java_ref, "group": "ly.count.sdk", "name": "java", "version": "26.8.1", "purl": java_ref}) + self.assertIn(java_ref, bom["dependencies"][0]["dependsOn"]) + self.assertEqual(bom["dependencies"][-1]["ref"], java_ref) + self.assertNotIn("project_path", json.dumps(bom)) + + def test_platform_specific_components_carry_no_hashes(self): + bom = copy.deepcopy(PLUGIN_BOM) + hashes = [{"alg": "SHA-256", "content": "c689de189abaa839eaf628d4b34aa176f6661e0d62ca918cb4af7544172e6a60"}] + bom["components"][0]["hashes"] = copy.deepcopy(hashes) + bom["components"].append({"type": "library", "bom-ref": "pkg:maven/org.openjfx/javafx-controls@21.0.5?type=jar", "group": "org.openjfx", "name": "javafx-controls", "version": "21.0.5", "hashes": copy.deepcopy(hashes), "purl": "pkg:maven/org.openjfx/javafx-controls@21.0.5?type=jar"}) + normalize(bom, self.ui, "26.8.1", "t", "https://maven.countly.com/", platform_groups=["org.openjfx"]) + self.assertEqual(bom["components"][0]["hashes"], hashes) + self.assertNotIn("hashes", bom["components"][1]) + self.assertEqual(bom["components"][1]["purl"], "pkg:maven/org.openjfx/javafx-controls@21.0.5?type=jar") + + def test_an_unpublished_module_stops_the_release(self): + bom = copy.deepcopy(PLUGIN_BOM) + bom["components"].append({"bom-ref": "x", "purl": "pkg:maven/x/app-java@unspecified?project_path=%3Aapp-java"}) + with self.assertRaises(SbomError): + normalize(bom, self.ui, "26.8.1", "t", "https://maven.countly.com/", {":sdk-java": self.java}) + + def test_place_sbom_writes_the_file_and_its_checksums(self): + staging = Path(tempfile.mkdtemp()) + source = staging / "bom.json" + source.write_bytes(json.dumps(PLUGIN_BOM).encode("utf-8")) + target = place_sbom(source, staging, self.java, "26.8.1", "t", "https://maven.countly.com/") + self.assertEqual(target, staging / "ly/count/sdk/java/26.8.1/java-26.8.1-cyclonedx.json") + self.assertEqual(json.loads(target.read_bytes())["metadata"]["component"]["name"], "java") + for suffix in [".md5", ".sha1", ".sha256", ".sha512"]: + self.assertTrue(Path(f"{target}{suffix}").is_file()) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/tests/test_sign.py b/.github/release/tests/test_sign.py new file mode 100644 index 000000000..f580c58e0 --- /dev/null +++ b/.github/release/tests/test_sign.py @@ -0,0 +1,74 @@ +import subprocess +import tempfile +import unittest +from pathlib import Path + +from sign import SigningError, export_public_key, sign_staging, signing_fingerprint + +RELEASE = "FC8DB0DE234A273BA45E562FB8C83A079A5BBD0C" +DRY_RUN = "1111111111111111111111111111111111111111" +OTHER = "2222222222222222222222222222222222222222" +SUBKEY = "9999999999999999999999999999999999999999" + + +def keyring(*fingerprints): + """A fake gpg whose secret keyring holds one primary key, with an encryption subkey, per fingerprint.""" + listing = "".join( + f"sec:u:4096:1:{fp[-16:]}:1:::u:::scESC:\nfpr:::::::::{fp}:\ngrp:::::::::{'A' * 40}:\n" + f"ssb:u:4096:1:{SUBKEY[-16:]}:1::::::e:\nfpr:::::::::{SUBKEY}:\n" + for fp in fingerprints + ) + + def fake_gpg(args, **kwargs): + return subprocess.CompletedProcess(args, 0, listing, "") + return fake_gpg + + +class SignTest(unittest.TestCase): + def test_signs_everything_but_checksums_with_the_chosen_key(self): + staging = Path(tempfile.mkdtemp()) + (staging / "a").mkdir() + for name in ["x.pom", "x.pom.sha1", "x.jar", "x.jar.md5"]: + (staging / "a" / name).write_bytes(b"x") + calls = [] + + def fake_gpg(args, **kwargs): + calls.append((args, kwargs.get("input"))) + Path(args[args.index("--output") + 1]).write_bytes(b"signature") + return subprocess.CompletedProcess(args, 0, "", "") + + signed = sign_staging(staging, "/tmp/gnupg", "secret", RELEASE, runner=fake_gpg) + self.assertEqual([path.name for path in signed], ["x.jar", "x.pom"]) + self.assertTrue((staging / "a/x.pom.asc").is_file()) + self.assertFalse((staging / "a/x.pom.sha1.asc").exists()) + self.assertTrue(all(passphrase == "secret" and "--passphrase-fd" in args for args, passphrase in calls)) + self.assertTrue(all(args[args.index("--local-user") + 1] == RELEASE + "!" for args, _ in calls)) + + def test_production_needs_the_release_key(self): + self.assertEqual(signing_fingerprint("/tmp/gnupg", RELEASE, production=True, runner=keyring(RELEASE)), RELEASE) + with self.assertRaises(SigningError): + signing_fingerprint("/tmp/gnupg", RELEASE, production=True, runner=keyring(OTHER)) + + def test_dry_run_refuses_the_release_key_and_needs_exactly_one_key(self): + self.assertEqual(signing_fingerprint("/tmp/gnupg", RELEASE, production=False, runner=keyring(DRY_RUN)), DRY_RUN) + for fingerprints in [(RELEASE,), (DRY_RUN, OTHER), ()]: + with self.subTest(fingerprints=fingerprints): + with self.assertRaises(SigningError): + signing_fingerprint("/tmp/gnupg", RELEASE, production=False, runner=keyring(*fingerprints)) + + def test_exports_the_public_key(self): + target = Path(tempfile.mkdtemp()) / "key.asc" + calls = [] + + def fake_gpg(args, **kwargs): + calls.append(args) + return subprocess.CompletedProcess(args, 0, "-----BEGIN PGP PUBLIC KEY BLOCK-----\nabc\n", "") + + export_public_key("/tmp/gnupg", DRY_RUN, target, runner=fake_gpg) + self.assertTrue(target.read_bytes().startswith(b"-----BEGIN PGP PUBLIC KEY BLOCK-----")) + self.assertIn("--export", calls[0]) + self.assertEqual(calls[0][-1], DRY_RUN) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/tests/test_sources.py b/.github/release/tests/test_sources.py new file mode 100644 index 000000000..58ea2cad1 --- /dev/null +++ b/.github/release/tests/test_sources.py @@ -0,0 +1,63 @@ +import re +import tempfile +import unittest +from pathlib import Path + +from plan import plan_for_tag +from sources import check_version_sources +from tests.support import repository_config + +BUILD_GRADLE = 'allprojects {\n ext.CLY_VERSION = "{version}"\n ext.POWERMOCK_VERSION = "1.7.4"\n}\n' +CONFIG_JAVA = 'public class Config {\n protected String sdkVersion = "{version}";\n}\n' + + +class SourcesTest(unittest.TestCase): + def setUp(self): + self.config = repository_config() + self.root = Path(tempfile.mkdtemp()) + + def write(self, relative, text): + """Writes a file of the fake checkout.""" + path = self.root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(text.encode("utf-8")) + + def write_sdk(self, properties_version, gradle_version, java_version, changelog): + """Writes the three version sources and the changelog.""" + self.write("gradle.properties", f"VERSION_NAME={properties_version}\nGROUP=ly.count.sdk\n") + self.write("build.gradle", BUILD_GRADLE.replace("{version}", gradle_version)) + self.write("sdk-java/src/main/java/ly/count/sdk/java/Config.java", CONFIG_JAVA.replace("{version}", java_version)) + self.write("CHANGELOG.md", changelog) + + def test_matching_sources(self): + self.write_sdk("26.8.1", "26.8.1", "26.8.1", "## 26.8.1\n* Fixed things.\n") + self.assertEqual(check_version_sources(self.root, plan_for_tag(self.config, "26.8.1"), self.config), []) + + def test_mismatches_are_reported_once(self): + self.write_sdk("26.8.1", "26.8.0", "26.8.2", "## XX.XX.XX\n") + problems = check_version_sources(self.root, plan_for_tag(self.config, "26.8.1"), self.config) + self.assertEqual(problems, [ + "build.gradle: version is 26.8.0, the tag says 26.8.1", + "sdk-java/src/main/java/ly/count/sdk/java/Config.java: version is 26.8.2, the tag says 26.8.1", + "CHANGELOG.md: no '## 26.8.1' heading", + ]) + + def test_candidates_need_no_changelog_heading(self): + self.write_sdk("26.8.1-rc1", "26.8.1-rc1", "26.8.1-rc1", "## XX.XX.XX\n") + self.assertEqual(check_version_sources(self.root, plan_for_tag(self.config, "26.8.1-rc1"), self.config), []) + + def test_windows_line_endings_are_accepted(self): + self.write_sdk("26.8.1\r", "26.8.1", "26.8.1", "## 26.8.1\r\n") + self.assertEqual(check_version_sources(self.root, plan_for_tag(self.config, "26.8.1"), self.config), []) + + def test_every_version_source_is_found_in_this_checkout(self): + root = Path(__file__).resolve().parents[3] + for spec in self.config["artifacts"].values(): + for source in spec["versionSources"]: + with self.subTest(file=source["file"]): + text = (root / source["file"]).read_text(encoding="utf-8") + self.assertIsNotNone(re.search(source["regex"], text, re.MULTILINE)) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/tests/test_upload.py b/.github/release/tests/test_upload.py new file mode 100644 index 000000000..45740ef98 --- /dev/null +++ b/.github/release/tests/test_upload.py @@ -0,0 +1,82 @@ +import datetime +import tempfile +import unittest +from pathlib import Path + +from index import rewrite_index +from plan import plan_for_tag +from tests.builders import write_artifact +from tests.support import FakeBucket, repository_config +from upload import IMMUTABLE, UploadError, content_type, upload_order, upload_release + +POM = "ly/count/sdk/java/26.8.1/java-26.8.1.pom" +JAR = "ly/count/sdk/java/26.8.1/java-26.8.1.jar" + + +def quiet(line): + """Swallows log lines.""" + + +class UploadTest(unittest.TestCase): + def setUp(self): + self.config = repository_config() + self.staging = Path(tempfile.mkdtemp()) + self.plan = plan_for_tag(self.config, "26.8.1") + + def stage(self): + """Stages both artifacts of the tag and returns the folder of java.""" + folders = [write_artifact(self.staging, artifact, self.plan.version) for artifact in self.plan.artifacts] + return folders[0] + + def test_pom_family_goes_last(self): + names = ["a.pom", "a.pom.asc", "a.pom.sha1", "a.jar", "a.jar.sha1", "a.module"] + self.assertEqual(upload_order(names, "a.pom"), ["a.jar", "a.jar.sha1", "a.module", "a.pom.asc", "a.pom.sha1", "a.pom"]) + + def test_upload_creates_then_skips_identical_files(self): + self.stage() + bucket = FakeBucket() + upload_release(bucket, self.staging, self.plan, log=quiet) + stored = dict(bucket.objects) + java_calls = [key for key in bucket.calls if key.startswith("ly/count/sdk/java/")] + self.assertEqual(java_calls[-1], POM) + self.assertEqual(bucket.calls[-1], "ly/count/sdk/java-ui/26.8.1/java-ui-26.8.1.pom") + self.assertEqual(stored[POM][2], IMMUTABLE) + self.assertEqual(stored[JAR][1], "application/java-archive") + upload_release(bucket, self.staging, self.plan, log=quiet) + self.assertEqual(bucket.objects, stored) + + def test_rerun_keeps_signatures_from_the_first_attempt(self): + folder = self.stage() + signature = folder / "java-26.8.1.pom.asc" + signature.write_bytes(b"first attempt") + bucket = FakeBucket() + upload_release(bucket, self.staging, self.plan, log=quiet) + signature.write_bytes(b"second attempt") + upload_release(bucket, self.staging, self.plan, log=quiet) + self.assertEqual(bucket.objects[POM + ".asc"][0], b"first attempt") + + def test_different_bytes_stop_the_upload(self): + folder = self.stage() + bucket = FakeBucket() + upload_release(bucket, self.staging, self.plan, log=quiet) + (folder / "java-26.8.1.module").write_bytes(b"rebuilt") + with self.assertRaises(UploadError): + upload_release(bucket, self.staging, self.plan, log=quiet) + + def test_interrupted_upload_is_never_listed(self): + self.stage() + bucket = FakeBucket(fail_on=POM) + with self.assertRaises(RuntimeError): + upload_release(bucket, self.staging, self.plan, log=quiet) + self.assertIn(JAR, bucket.objects) + self.assertEqual(rewrite_index(bucket, "ly.count.sdk", "java", datetime.datetime(2026, 10, 27), log=quiet), []) + + def test_content_types(self): + cases = [("a.pom", "application/xml"), ("a.pom.sha1", "text/plain; charset=utf-8"), ("a.jar.asc", "application/pgp-signature"), ("a.module", "application/json"), ("a-cyclonedx.json", "application/json"), ("a.jar", "application/java-archive"), ("a.bin", "application/octet-stream")] + for name, kind in cases: + with self.subTest(name=name): + self.assertEqual(content_type(name), kind) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/tests/test_verify.py b/.github/release/tests/test_verify.py new file mode 100644 index 000000000..1840bc235 --- /dev/null +++ b/.github/release/tests/test_verify.py @@ -0,0 +1,65 @@ +import hashlib +import subprocess +import tempfile +import unittest +from pathlib import Path + +from plan import plan_for_tag +from tests.support import FakeWeb, repository_config +from verify import verify_public, verify_signatures + +BASE = "https://maven.test/" +POM = "ly/count/sdk/java/26.8.1/java-26.8.1.pom" + + +class VerifyTest(unittest.TestCase): + def setUp(self): + self.config = repository_config() + self.plan = plan_for_tag(self.config, "26.8.1") + self.work = Path(tempfile.mkdtemp()) + self.manifest = {"tag": "26.8.1", "commit": "c" * 40, "files": [ + {"path": POM, "size": 3, "sha256": hashlib.sha256(b"pom").hexdigest()}, + {"path": POM + ".sha1", "size": 4, "sha256": hashlib.sha256(b"sha1").hexdigest()}, + ]} + self.web = FakeWeb({ + BASE + POM: b"pom", + BASE + POM + ".sha1": b"sha1", + BASE + POM + ".asc": b"signature", + BASE + "ly/count/sdk/java/maven-metadata.xml": b"26.8.1", + BASE + "ly/count/sdk/java-ui/maven-metadata.xml": b"26.8.1", + }) + + def test_published_release_passes_and_keeps_the_signatures(self): + self.assertEqual(verify_public(self.manifest, self.plan, BASE, "1", self.work, fetcher=self.web), []) + self.assertEqual((self.work / (POM + ".asc")).read_bytes(), b"signature") + self.assertTrue(all("?nocache=1" in url for url in self.web.requests)) + + def test_problems_are_reported(self): + self.web.pages[BASE + POM] = b"changed" + del self.web.pages[BASE + POM + ".asc"] + self.web.pages[BASE + "ly/count/sdk/java-ui/maven-metadata.xml"] = b"26.8.0" + self.assertEqual(verify_public(self.manifest, self.plan, BASE, "1", self.work, fetcher=self.web), [ + f"changed {POM}", + f"missing {POM}.asc", + "ly.count.sdk:java-ui index does not list 26.8.1", + ]) + + def test_candidate_must_stay_unlisted(self): + plan = plan_for_tag(self.config, "26.8.1-rc1") + web = FakeWeb({BASE + "ly/count/sdk/java/maven-metadata.xml": b"26.8.1-rc1"}) + self.assertEqual(verify_public({"files": []}, plan, BASE, "1", self.work, fetcher=web), ["ly.count.sdk:java index lists the release candidate 26.8.1-rc1"]) + + def test_bad_signatures_are_reported(self): + (self.work / "a").mkdir() + for name in ["x.pom", "x.pom.asc", "y.pom", "y.pom.asc"]: + (self.work / "a" / name).write_bytes(b"x") + + def fake_gpg(args, **kwargs): + bad = "--verify" in args and args[-1].endswith("y.pom") + return subprocess.CompletedProcess(args, 1 if bad else 0, b"", b"") + + self.assertEqual(verify_signatures(self.work, Path("key.asc"), runner=fake_gpg), ["bad signature a/y.pom.asc"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/release/upload.py b/.github/release/upload.py new file mode 100644 index 000000000..4bfb96824 --- /dev/null +++ b/.github/release/upload.py @@ -0,0 +1,59 @@ +"""Uploads a signed release create-only, in an order that keeps the repository consistent at every moment.""" + +import hashlib +from pathlib import Path + +IMMUTABLE = "public, max-age=31536000, immutable" +CONTENT_TYPES = ( + ((".md5", ".sha1", ".sha256", ".sha512"), "text/plain; charset=utf-8"), + ((".asc",), "application/pgp-signature"), + ((".pom", ".xml"), "application/xml"), + ((".jar",), "application/java-archive"), + ((".module", ".json"), "application/json"), +) + + +class UploadError(Exception): + """A stored object differs from the file that should be uploaded; the version number is burned.""" + + +def content_type(name): + """Content type of a repository file, chosen by its extension.""" + for suffixes, kind in CONTENT_TYPES: + if name.endswith(suffixes): + return kind + return "application/octet-stream" + + +def upload_order(names, pom_name): + """Every file except the POM family first, then the POM's checksums and signature, the POM itself last.""" + family = sorted(name for name in names if name.startswith(pom_name) and name != pom_name) + others = sorted(name for name in names if not name.startswith(pom_name)) + return others + family + ([pom_name] if pom_name in names else []) + + +def upload_folder(bucket, local_folder, remote_folder, pom_name, log=print): + """Uploads one version folder create-only. An identical stored file is skipped; a signature stored by an earlier + attempt is kept; any other difference raises UploadError.""" + names = sorted(path.name for path in Path(local_folder).iterdir() if path.is_file()) + for name in upload_order(names, pom_name): + path = Path(local_folder) / name + key = f"{remote_folder}/{name}" + if bucket.put_file(key, path, content_type(name), IMMUTABLE, create_only=True) == "created": + log(f"created {key}") + continue + stored = bucket.head(key) + if stored is not None and stored["etag"] == hashlib.md5(path.read_bytes()).hexdigest(): + log(f"identical {key}") + elif stored is not None and name.endswith(".asc"): + # Re-signing gives other bytes (a signature carries its creation time); the verify job checks the stored one. + log(f"kept the signature stored earlier {key}") + else: + raise UploadError(f"{key} already exists with different content") + + +def upload_release(bucket, staging_dir, plan, log=print): + """Uploads every artifact version folder of the plan.""" + for artifact in plan.artifacts: + folder = artifact.folder(plan.version) + upload_folder(bucket, Path(staging_dir) / folder, folder, f"{artifact.base_name(plan.version)}.pom", log) diff --git a/.github/release/verify.py b/.github/release/verify.py new file mode 100644 index 000000000..4fa860b1e --- /dev/null +++ b/.github/release/verify.py @@ -0,0 +1,65 @@ +"""Checks a release from the public address, the way a customer's build sees it.""" + +import hashlib +import subprocess +import tempfile +import urllib.error +import urllib.request +from pathlib import Path + +from layout import is_checksum + + +def fetch(url, timeout=60): + """Downloads a URL; returns None for 404 and raises for any other error.""" + request = urllib.request.Request(url, headers={"User-Agent": "countly-sdk-release"}) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.read() + except urllib.error.HTTPError as error: + if error.code == 404: + return None + raise + + +def verify_public(manifest, plan, base_url, nocache, work_dir, fetcher=fetch): + """Problems found when downloading the release: missing or changed files, missing signatures, wrong index. Downloaded files and signatures are kept in work_dir.""" + problems = [] + base = base_url.rstrip("/") + "/" + for entry in manifest["files"]: + body = fetcher(f"{base}{entry['path']}?nocache={nocache}") + if body is None: + problems.append(f"missing {entry['path']}") + continue + if hashlib.sha256(body).hexdigest() != entry["sha256"]: + problems.append(f"changed {entry['path']}") + if is_checksum(entry["path"]): + continue + signature = fetcher(f"{base}{entry['path']}.asc?nocache={nocache}") + if signature is None: + problems.append(f"missing {entry['path']}.asc") + continue + target = Path(work_dir) / entry["path"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(body) + Path(f"{target}.asc").write_bytes(signature) + for artifact in plan.artifacts: + index = fetcher(f"{base}{artifact.group_path}/{artifact.artifact}/maven-metadata.xml?nocache={nocache}") + listed = index is not None and f"{plan.version}".encode("utf-8") in index + if plan.listed and not listed: + problems.append(f"{artifact.coordinates} index does not list {plan.version}") + if not plan.listed and listed: + problems.append(f"{artifact.coordinates} index lists the release candidate {plan.version}") + return problems + + +def verify_signatures(work_dir, public_key, runner=subprocess.run): + """Checks every signature under work_dir against the given public key only; returns problems.""" + home = tempfile.mkdtemp() + runner(["gpg", "--homedir", home, "--batch", "--import", str(public_key)], capture_output=True, check=True) + problems = [] + for signature in sorted(Path(work_dir).rglob("*.asc")): + result = runner(["gpg", "--homedir", home, "--batch", "--verify", str(signature), str(signature.with_suffix(""))], capture_output=True) + if result.returncode != 0: + problems.append(f"bad signature {signature.relative_to(work_dir).as_posix()}") + return problems diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..765f30dcc --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,34 @@ +name: "CodeQL" + +on: + push: + branches: [master, staging] + pull_request: + branches: [master, staging] + schedule: + - cron: '18 18 * * 1' + +permissions: {} + +jobs: + analyze: + name: Analyze (java) + runs-on: ubuntu-24.04 + permissions: + actions: read + contents: read + security-events: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Initialize CodeQL + uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + with: + languages: java + # Without a build the scan needs no JDK or display, so the check a release requires cannot fail for build reasons. + build-mode: none + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + with: + category: "/language:java" diff --git a/.github/workflows/publishing.yml b/.github/workflows/publishing.yml deleted file mode 100644 index 5daee1dab..000000000 --- a/.github/workflows/publishing.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Publish the Java SDK -on: - workflow_dispatch: - inputs: - deploy: - description: 'Deploy?' - required: true - type: boolean -jobs: - build: - name: Build and Test - if: ${{ inputs.deploy }} - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up JDK 8 - uses: actions/setup-java@v3 - with: - java-version: '8' - distribution: 'corretto' - - - name: Build and Test with Gradle - run: ./gradlew build - - - name: Publish SDK Library to Maven Central - run: ./gradlew publishAllPublicationsToMavenCentralRepository --no-configuration-cache - env: - ORG_GRADLE_PROJECT_mavenCentralUsername: ${{ secrets.MAVEN_USERNAME }} - ORG_GRADLE_PROJECT_mavenCentralPassword: ${{ secrets.MAVEN_PASSWORD }} - ORG_GRADLE_PROJECT_signingInMemoryKey: ${{ secrets.GPG_KEY }} - ORG_GRADLE_PROJECT_signingInMemoryKeyId: ${{ secrets.GPG_KEY_ID }} - ORG_GRADLE_PROJECT_signingInMemoryKeyPassword: ${{ secrets.GPG_KEY_PASSWORD }} diff --git a/.github/workflows/release-tools.yml b/.github/workflows/release-tools.yml new file mode 100644 index 000000000..8986f3bbb --- /dev/null +++ b/.github/workflows/release-tools.yml @@ -0,0 +1,88 @@ +name: Release tools + +on: + pull_request: + push: + branches: [master, staging] + +permissions: + contents: read + +jobs: + test: + name: Release tools tests + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Unit tests + working-directory: .github/release + run: python3 -m unittest discover -s tests -t . -v + + lint: + name: Release workflow lint + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: actionlint + run: | + curl -sSfL -o "$RUNNER_TEMP/actionlint.tar.gz" https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz + echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 $RUNNER_TEMP/actionlint.tar.gz" | sha256sum -c - + tar -xzf "$RUNNER_TEMP/actionlint.tar.gz" -C "$RUNNER_TEMP" actionlint + # actionlint 1.7.12 predates the concurrency queue key GitHub added in May 2026 (rhysd/actionlint#654). + "$RUNNER_TEMP/actionlint" -ignore 'unexpected key "queue" for "concurrency" section' .github/workflows/release.yml .github/workflows/release-tools.yml + - name: zizmor + run: | + curl -sSfL -o "$RUNNER_TEMP/zizmor.tar.gz" https://github.com/zizmorcore/zizmor/releases/download/v1.30.1/zizmor-x86_64-unknown-linux-gnu.tar.gz + echo "e65324f4430c2717591937edcec90ccbefaf14c174f8ec9415e03ca875b46e1a $RUNNER_TEMP/zizmor.tar.gz" | sha256sum -c - + tar -xzf "$RUNNER_TEMP/zizmor.tar.gz" -C "$RUNNER_TEMP" zizmor + # The informational findings are expansions of the plan's Gradle task lists, which come from config.json in the same checkout. + "$RUNNER_TEMP/zizmor" --offline --min-severity low .github/workflows/release.yml .github/workflows/release-tools.yml + + tags: + name: Tags this branch releases + runs-on: ubuntu-24.04 + outputs: + tags: ${{ steps.tags.outputs.tags }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Read the version files + id: tags + env: + BRANCH: ${{ github.base_ref || github.ref_name }} + run: python3 .github/release/release.py checkout-tags --branch "$BRANCH" + + checks: + name: Release checks (${{ matrix.tag }}) + needs: tags + if: needs.tags.outputs.tags != '[]' + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + tag: ${{ fromJSON(needs.tags.outputs.tags) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 + with: + distribution: corretto + java-version: '17' + - uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0 + - name: Plan + id: plan + env: + TAG: ${{ matrix.tag }} + run: python3 .github/release/release.py plan --tag "$TAG" --target test --out plan.json + - name: Publish into the staging folder + run: ./gradlew --no-configuration-cache ${{ steps.plan.outputs.publish_tasks }} -PRELEASE_SIGNING_ENABLED=false + - name: Stage the release and check its files + run: python3 .github/release/release.py stage --plan plan.json --staging build/release-staging --commit "$GITHUB_SHA" --target test --manifest release-manifest.json + - name: Check what the release promises integrators + run: python3 .github/release/release.py contract --plan plan.json --staging build/release-staging diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..d95216c38 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,244 @@ +name: Release + +on: + release: + types: [published] + workflow_dispatch: + inputs: + target: + description: "production publishes to maven.countly.com; test is a dry run against maven-test.countly.com" + type: choice + options: [production, test] + default: production + +permissions: {} + +concurrency: + group: maven-publish + cancel-in-progress: false + queue: max + +env: + TAG: ${{ github.ref_name }} + TARGET: ${{ inputs.target || 'production' }} + +jobs: + plan: + name: Plan and check + runs-on: ubuntu-24.04 + permissions: + contents: read + checks: read + outputs: + environment: ${{ steps.plan.outputs.environment }} + check_tasks: ${{ steps.plan.outputs.check_tasks }} + publish_tasks: ${{ steps.plan.outputs.publish_tasks }} + published_modules: ${{ steps.plan.outputs.published_modules }} + steps: + - name: Only tags can be released + if: github.ref_type != 'tag' + run: | + echo "::error::Run the release workflow on a tag, not on a branch." + exit 1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Read the GitHub release + id: release + if: env.TARGET == 'production' + env: + GH_TOKEN: ${{ github.token }} + run: | + prerelease=$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isPrerelease --jq .isPrerelease) + echo "prerelease=$prerelease" >> "$GITHUB_OUTPUT" + - name: Plan + id: plan + run: python3 .github/release/release.py plan --tag "$TAG" --target "$TARGET" --commit "$GITHUB_SHA" --prerelease "${{ steps.release.outputs.prerelease || 'unknown' }}" --out plan.json + - name: The tag commit is on the release branch + if: env.TARGET == 'production' + run: python3 .github/release/release.py check-branch --plan plan.json --commit "$GITHUB_SHA" + - name: Version numbers match the tag + run: python3 .github/release/release.py check-sources --plan plan.json + - name: The version is not published yet + run: python3 .github/release/release.py check-absent --plan plan.json --target "$TARGET" --nocache "${{ github.run_id }}" + - name: Required checks passed on the tag commit + if: env.TARGET == 'production' + env: + GH_TOKEN: ${{ github.token }} + run: | + gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/check-runs?per_page=100" > check-runs.json + python3 .github/release/release.py check-required --check-runs check-runs.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: plan + path: plan.json + retention-days: 7 + if-no-files-found: error + + build: + name: Build and check + needs: plan + runs-on: ubuntu-24.04 + # A hung test or clean-project build would otherwise hold the release queue for GitHub's 6-hour default. + timeout-minutes: 60 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: plan + - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 + with: + distribution: corretto + java-version: | + 8 + 17 + - name: Record the JDK and Maven + run: | + echo '### JDK, Java 8 runtime and Maven' >> "$GITHUB_STEP_SUMMARY" + { java -version; "$JAVA_HOME_8_X64/bin/java" -version; mvn -v; } 2>&1 | sed 's/^/ /' >> "$GITHUB_STEP_SUMMARY" + - uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0 + with: + cache-disabled: true + - name: Make sure a virtual display for the JavaFX tests exists + run: command -v xvfb-run > /dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq xvfb; } + - name: Test and check the modules + run: xvfb-run -a ./gradlew --no-daemon --no-build-cache --no-configuration-cache ${{ needs.plan.outputs.check_tasks }} + - name: Publish into the staging folder + run: ./gradlew --no-daemon --no-build-cache --no-configuration-cache ${{ needs.plan.outputs.publish_tasks }} -PRELEASE_SIGNING_ENABLED=false + - name: Stage the release and check its files + run: python3 .github/release/release.py stage --plan plan.json --staging build/release-staging --commit "$GITHUB_SHA" --target "$TARGET" --manifest release-manifest.json + - name: Check the SBOMs against the CycloneDX 1.6 schema + run: | + curl -sSfL -o "$RUNNER_TEMP/cyclonedx" https://github.com/CycloneDX/cyclonedx-cli/releases/download/v0.33.1/cyclonedx-linux-x64 + echo "bfc8b2538da86fe239bc53658bbb63c1c8c510a293c1e6891aa5bea5d3c58746 $RUNNER_TEMP/cyclonedx" | sha256sum -c - + chmod +x "$RUNNER_TEMP/cyclonedx" + python3 .github/release/release.py validate-sbom --plan plan.json --staging build/release-staging --cli "$RUNNER_TEMP/cyclonedx" + - name: Check what the release promises integrators + run: python3 .github/release/release.py contract --plan plan.json --staging build/release-staging + - name: Clean projects build against the staged release + run: python3 .github/release/release.py consumer --plan plan.json --staging build/release-staging --java8-home "$JAVA_HOME_8_X64" --work "$RUNNER_TEMP/consumers" + - name: Scan the published dependencies + env: + OSV_FAIL_ON: high + OSV_BLOCKING_SCOPES: published + run: | + ./gradlew -q --no-daemon --no-configuration-cache --init-script .github/scripts/dependency-report.init.gradle "-DpublishedModules=${{ needs.plan.outputs.published_modules }}" printResolvedDependencies > resolved-dependencies.txt + python3 .github/scripts/osv_scan.py < resolved-dependencies.txt + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: staged-release + path: | + build/release-staging + release-manifest.json + retention-days: 7 + if-no-files-found: error + + publish: + name: Approve, sign and upload + needs: [plan, build] + runs-on: ubuntu-24.04 + environment: ${{ needs.plan.outputs.environment }} + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: plan + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: staged-release + - name: The files are exactly what the build produced + run: python3 .github/release/release.py verify-manifest --manifest release-manifest.json --staging build/release-staging + - name: Sign and check the signatures + env: + GNUPGHOME: ${{ runner.temp }}/gnupg + SIGNING_KEY: ${{ secrets.SIGNING_KEY }} + SIGNING_KEY_PASSPHRASE: ${{ secrets.SIGNING_KEY_PASSPHRASE }} + run: | + mkdir -m 700 "$GNUPGHOME" + printf '%s\n' "$SIGNING_KEY" | gpg --batch --import + python3 .github/release/release.py sign --staging build/release-staging --target "$TARGET" --public-key-out signing-public-key.asc + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: signing-public-key + path: signing-public-key.asc + retention-days: 7 + overwrite: true + if-no-files-found: error + - name: Upload to R2 and rewrite the index + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + R2_ACCOUNT_ID: ${{ vars.R2_ACCOUNT_ID }} + R2_BUCKET: ${{ vars.R2_BUCKET }} + run: | + python3 .github/release/release.py upload --plan plan.json --staging build/release-staging + python3 .github/release/release.py index --plan plan.json + - name: Remove the signing key + if: always() + env: + GNUPGHOME: ${{ runner.temp }}/gnupg + run: | + gpgconf --kill gpg-agent || true + rm -rf "$GNUPGHOME" + + verify: + name: Verify from the public address + needs: publish + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: plan + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: staged-release + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: signing-public-key + - name: Files, signatures and index + run: python3 .github/release/release.py verify-public --plan plan.json --manifest release-manifest.json --target "$TARGET" --nocache "${{ github.run_id }}-${{ github.run_attempt }}" --work "${{ runner.temp }}/downloaded" --public-key signing-public-key.asc + + announce: + name: Announce + needs: verify + if: github.event_name == 'release' && !github.event.release.prerelease + runs-on: ubuntu-24.04 + permissions: {} + steps: + - name: Slack + uses: slackapi/slack-github-action@007b2c3c751a190b6f0f040e47ed024deaa72844 # v1.23.0 + with: + payload: | + { + "repository": "${{ github.repository }}", + "tag_name": "${{ github.event.release.tag_name }}", + "actor": "${{ github.actor }}", + "body": ${{ toJSON(github.event.release.body) }}, + "html_url": "${{ github.event.release.html_url }}" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE }} + - name: Discord + uses: sarisia/actions-status-discord@9904e3130b8905d5b973df25623f17672dcb3466 # v1.13.0 + with: + webhook: ${{ secrets.DISCORD_WEBHOOK_URL }} + nodetail: true + title: New ${{ github.repository }} version ${{ github.event.release.tag_name }} published by ${{ github.actor }} + description: | + Release URL: ${{ github.event.release.html_url }} + Click [here](https://github.com/Countly/countly-server/blob/master/CHANGELOG.md) to view the change log. + `${{ github.event.release.body }}` diff --git a/.github/workflows/release_notice.yml b/.github/workflows/release_notice.yml deleted file mode 100644 index e17315a01..000000000 --- a/.github/workflows/release_notice.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Release Notice -on: - release: - types: [published] - workflow_dispatch: -jobs: - build: - runs-on: ubuntu-latest - steps: - # To check the github context - - name: Dump Github context - env: - GITHUB_CONTEXT: ${{ toJSON(github) }} - run: echo "$GITHUB_CONTEXT" - - name: Send custom JSON data to Slack workflow - id: slack - uses: slackapi/slack-github-action@v1.23.0 - with: - # This data can be any valid JSON from a previous step in the GitHub Action - payload: | - { - "repository": "${{ github.repository }}", - "tag_name": "${{ github.event.release.tag_name }}", - "actor": "${{ github.actor }}", - "body": ${{ toJSON(github.event.release.body) }}, - "html_url": "${{ github.event.release.html_url }}" - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE }} - - name: Send custom JSON data to Discord - uses: sarisia/actions-status-discord@v1.13.0 - with: - webhook: ${{ secrets.DISCORD_WEBHOOK_URL }} - nodetail: true - title: New ${{ github.repository }} version ${{ github.event.release.tag_name }} published by ${{ github.actor }} - description: | - Release URL: ${{ github.event.release.html_url }} - Click [here](https://github.com/Countly/countly-server/blob/master/CHANGELOG.md) to view the change log. - `${{ github.event.release.body }}` diff --git a/CHANGELOG.md b/CHANGELOG.md index f088611b1..81a5fe715 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,25 @@ +## XX.XX.XX +* !! Major breaking change !! New versions of `ly.count.sdk:java` and `ly.count.sdk:java-ui` are published at `https://maven.countly.com` instead of Maven Central. Add the repository once. + Gradle, next to `mavenCentral()` in `dependencyResolutionManagement` of `settings.gradle` (or in `allprojects` of the root `build.gradle` in older projects): + ```groovy + maven { + url = 'https://maven.countly.com' + content { includeGroupByRegex 'ly\\.count\\..*' } + } + ``` + Maven, in `pom.xml`: + ```xml + + + countly + https://maven.countly.com + true + false + + + ``` + Versions released before this one stay available on Maven Central. The files are signed with the same key as before (`FC8DB0DE234A273BA45E562FB8C83A079A5BBD0C`). + ## 26.8.0 * Added support for the Content feature, accessible through the "Countly.instance().content()" interface: * "enterContentZone" / "exitContentZone" for starting and stopping periodic content fetching diff --git a/build.gradle b/build.gradle index b8f1479b3..7f224fc92 100644 --- a/build.gradle +++ b/build.gradle @@ -4,6 +4,10 @@ buildscript { repositories { google() mavenCentral() + // CycloneDX publishes its Gradle plugin 3.x only on the Gradle Plugin Portal. + gradlePluginPortal { + content { includeGroup 'org.cyclonedx' } + } } dependencies { // The publish plugin is loaded here, once, and only when a publish task is requested. Loading it @@ -13,6 +17,25 @@ buildscript { if (gradle.startParameter.taskNames.any { it.toLowerCase().contains("publish") }) { // This requires minimum java 11 to work classpath 'com.vanniktech:gradle-maven-publish-plugin:0.28.0' + classpath 'org.cyclonedx:cyclonedx-gradle-plugin:3.4.1' + } + } +} + +// Publishing modules: the local staging repository release.yml uploads from, and the SBOM of the published runtime classpath. +subprojects { + plugins.withId('com.vanniktech.maven.publish') { + publishing { + repositories { + maven { + name = 'releaseStaging' + url = rootProject.file('build/release-staging') + } + } + } + apply plugin: 'org.cyclonedx.bom' + tasks.named('cyclonedxDirectBom').configure { + includeConfigs = ['runtimeClasspath'] } } } diff --git a/sdk-java/build.gradle b/sdk-java/build.gradle index 365bf0758..6feb81ecc 100644 --- a/sdk-java/build.gradle +++ b/sdk-java/build.gradle @@ -10,6 +10,13 @@ java { targetCompatibility = JavaVersion.VERSION_1_8 } +// javac 8 has no --release flag; on a newer JDK it also stops the use of Java 9+ APIs, which the target alone allows. +if (JavaVersion.current().isJava9Compatible()) { + tasks.withType(JavaCompile).configureEach { + options.release = 8 + } +} + dependencies { implementation 'org.json:json:20250517' implementation 'com.google.code.findbugs:jsr305:3.0.2'