From c722f13f6d53fb0980dd394bc8302dc6dcb83982 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Thu, 19 Nov 2026 21:13:11 +0530 Subject: [PATCH 1/5] feat(dedup): fingerprint only the app's own classes, found from launch With no KEPLOY_JAVA_CLASS_DIRS, a Jib image (java -cp @jib-classpath-file Main) gave the agent no class dirs, so it indexed the whole classpath: on Apache Fineract 112k classes, about 7k of them the app's. A shaded jar was indexed whole. Library code in every fingerprint made duplicate tests look different (dedup kept 41% of Fineract's tests), and the warmup over all of it ran for 54 s inside the first START, past the time Keploy waits for its answer. The agent now derives the app's classes from how the JVM was launched. The dir or jar that holds the main class is the app's own and counts whole, so generated code outside its package still counts. Other jars give only the classes under the main class's package root: its package cut to three parts, or a longer Maven groupId that holds it (a shorter one never widens it). - java -cp ... Main (Jib, or any classpath): the classpath's dirs and the jar that holds the main class, and the classes under the root in its other jars. - A Spring Boot jar or war, or the dir it was extracted to: BOOT-INF/classes, and the classes under the root in BOOT-INF/lib's jars. Each lib is read in place from the jar mapped read-only, by its central directory, so only those classes are inflated and no lib is copied into the app's heap. A thin jar counts by its Start-Class. - A Vert.x app: its Main-Verticle, or the verticle its command runs, is the main class. - Any other jar: all of it, or only the classes under the root when it is shaded (it holds other projects' pom.properties), and the classes under the root in the jars its manifest's Class-Path names. A jar shaded of libraries with no pom.properties counts whole, as before. A launch it cannot read is not guessed at: a main class that is not in the jar or classpath launched (a launcher stub, as Quarkus and Maven's test forks run), a known server, framework or test runner launcher (Tomcat, Jetty, WildFly, Play, Helidon MP, Karaf, Felix, Equinox, Maven, Gradle), or java -m. A launcher not on that list is taken for the app. KEPLOY_JAVA_CLASS_DIRS and the build's target/classes still come first. When the derivation finds no class, the agent still falls back to the whole jar, then the whole classpath, but now logs that at WARNING with the reason and the fix, and skips the warmup: the warmup runs only over the classes found as the app's, as a library's static initializer can start threads or attach an agent. An index left empty is said at WARNING too, as no test's coverage is measured then. A dir it cannot read is passed over instead of failing the whole index, and a symlink to a class file is followed. The README says which classes count as the app's. Signed-off-by: slayerjain --- README.md | 47 +- .../main/java/io/keploy/dedup/AppClasses.java | 1031 +++++++++++++++++ .../io/keploy/dedup/KeployDedupAgent.java | 131 ++- .../java/io/keploy/dedup/AppClassesTest.java | 834 +++++++++++++ .../java/io/keploy/dedup/AppIndexTest.java | 356 ++++++ 5 files changed, 2349 insertions(+), 50 deletions(-) create mode 100644 keploy-sdk/src/main/java/io/keploy/dedup/AppClasses.java create mode 100644 keploy-sdk/src/test/java/io/keploy/dedup/AppClassesTest.java create mode 100644 keploy-sdk/src/test/java/io/keploy/dedup/AppIndexTest.java diff --git a/README.md b/README.md index 8ebbf646..1b9ef46d 100644 --- a/README.md +++ b/README.md @@ -173,10 +173,53 @@ ceiling does not quietly fall behind again. - `KEPLOY_JACOCO_HOST`: JaCoCo TCP host used when the in-process runtime API is unavailable. Default: `127.0.0.1` - `KEPLOY_JACOCO_PORT`: JaCoCo TCP port used when the in-process runtime API is unavailable. Default: `36320` -- `KEPLOY_JAVA_CLASS_DIRS`: optional comma-separated class, jar, war, ear, or zip locations to analyze for executed lines when your build output lives outside the standard locations -- `KEPLOY_JAVA_CLASSPATH_FALLBACK`: scans the full classpath if standard class roots and the executable archive do not provide application classes. Default: `true` +- `KEPLOY_JAVA_CLASS_DIRS`: optional comma-separated class, jar, war, ear, or zip locations of the app's own classes. You rarely need it: see "Which classes count as the app's" below +- `KEPLOY_JAVA_CLASSPATH_FALLBACK`: scans the full classpath when neither the app's own classes nor the executable jar give any class. `false` leaves the index empty then, and no test's coverage is measured. Default: `true` - `KEPLOY_JAVA_DEDUP_DISABLED`: disables the Java dedup agent when set to `true`, `1`, or `yes` +## Which classes count as the app's + +Dedup fingerprints a test by the app's own code it runs, not by library code +(Spring, Hibernate, Jetty), which runs on nearly every request. The agent finds +the app's classes, in this order: + +1. `KEPLOY_JAVA_CLASS_DIRS`, and `target/classes` or `build/classes/java/main` + under the working dir. +2. From how the JVM was launched. The dir or jar that holds the main class is + the app's own and counts whole, so generated code outside its package + (OpenAPI, jOOQ, protobuf) counts too. Other jars give only the classes under + the main class's package root: its package cut to three parts + (`org.apache.fineract` for `org.apache.fineract.ServerApplication`), or a + longer Maven groupId that the package is in. + - `java -cp ... Main` (Jib's `/app/classes` and `/app/libs`, or any other + classpath): every class in the classpath's directories and in the jar + that holds the main class, and the classes under the root in its other + jars. + - A Spring Boot jar or war, or the dir it was extracted to: every class in + `BOOT-INF/classes`, and the classes under the root in `BOOT-INF/lib`'s + jars. A thin jar counts by its `Start-Class`. + - A Vert.x app: its `Main-Verticle`, or the verticle its command runs, is + the main class. + - Any other jar: all of it, and the classes under the root in the jars its + manifest's `Class-Path` names. A shaded jar (one that holds other + projects' `META-INF/maven` metadata) gives only the classes under the root. + A jar shaded of libraries that carry no such metadata (as Gradle's shadow + often is) counts whole, library classes too. + A launch the agent cannot read this way is not guessed at: a main class in + none of the jars or dirs launched (a launcher stub, as Quarkus runs), a + known server's, framework's or test runner's launcher (Tomcat, Jetty, + WildFly, Play, Helidon MP, Karaf, Felix, Equinox, Maven, Gradle), or + `java -m`. The list is of known launchers: a launcher it does not know is + taken for the app, so set `KEPLOY_JAVA_CLASS_DIRS` for one. +3. Failing those, the whole executable jar, then the whole classpath + (`KEPLOY_JAVA_CLASSPATH_FALLBACK`). The agent then logs a WARNING: library + code makes duplicate tests look different, so dedup keeps more tests than + it should, and the startup warmup is skipped. When even those give no + class, the WARNING says no test's coverage is measured. Either way, set + `KEPLOY_JAVA_CLASS_DIRS`. + +The agent logs where its classes came from, and how many, at the first test. + ## Sample For a working reference, see the Java dedup sample in `keploy/samples-java`: diff --git a/keploy-sdk/src/main/java/io/keploy/dedup/AppClasses.java b/keploy-sdk/src/main/java/io/keploy/dedup/AppClasses.java new file mode 100644 index 00000000..3755da29 --- /dev/null +++ b/keploy-sdk/src/main/java/io/keploy/dedup/AppClasses.java @@ -0,0 +1,1031 @@ +package io.keploy.dedup; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.Buffer; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarInputStream; +import java.util.jar.Manifest; +import java.util.regex.Pattern; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; + +/** + * AppClasses tells the app's own classes from its libraries' by how the JVM + * was launched, so that dedup fingerprints, and warms up, only the app's. + *

+ * Library code (Spring, Hibernate, Jetty, Jackson) runs on nearly every + * request, so in a fingerprint it makes tests that run the same app code look + * different. On a large app it is also most of the classpath (Fineract: about + * 7k own classes in 112k), and an index of all of it made the warmup, the + * per-test dump and the class upload too big to finish. + *

+ * The app's own code is where its main class is: the classpath dir or jar + * that holds it is taken whole, so generated code outside the main class's + * package (OpenAPI, jOOQ, protobuf) still counts. The other jars give only + * the classes under the package root of the main class + * ({@link #root(String, Collection)}): + *

+ * A jar counts by the classes it holds, not by its Maven groupId: Gradle + * builds write no {@code pom.properties}, so the app's own module jars often + * carry none. + *

+ * A launch it cannot read this way fails with a reason instead of guessing: + * a main class that is not in the jar or classpath it names (a launcher + * stub, as Quarkus and Maven's test forks run), or a known launcher's main + * class (Tomcat, Jetty, WildFly, Play, Helidon MP, Karaf), whose package is + * not the app's. Vert.x's launcher runs the app's verticle, which is then + * the main class. + */ +final class AppClasses { + + private static final Pattern CLASS_NAME = + Pattern.compile("[A-Za-z_$][A-Za-z0-9_$]*(\\.[A-Za-z_$][A-Za-z0-9_$]*)*"); + private static final String MAVEN_DIR = "META-INF/maven/"; + private static final String BOOT_LAUNCHER = "org.springframework.boot.loader."; + private static final String BOOT_LAUNCHER_PATH = "org/springframework/boot/loader/"; + // The package root is at most this many parts of the main class's + // package, unless a Maven groupId on the classpath names a longer one. + private static final int ROOT_PARTS = 3; + // Main classes of servers, frameworks, build tools and test runners, + // which load the app themselves: their package is never the app's. A + // name that ends in a dot is a package; any other is one class. + private static final String[] LAUNCHERS = { + "org.apache.catalina.startup.", + "org.eclipse.jetty.start.", + "org.jboss.modules.", + "io.quarkus.bootstrap.", + "io.vertx.core.Launcher", + "io.vertx.launcher.", + "play.core.server.", + "io.helidon.microprofile.cdi.", + "org.apache.karaf.main.", + "org.apache.felix.main.", + "org.eclipse.equinox.launcher.", + "com.intellij.rt.", + "org.apache.maven.surefire.", + "org.codehaus.plexus.classworlds.", + "org.gradle.", + "worker.org.gradle.", + "org.junit.platform.console.", + }; + // Vert.x's launchers, which run the verticle the manifest's + // Main-Verticle or the command names: that class is the app's. + private static final String VERTX_LAUNCHER = "io.vertx.core.Launcher"; + private static final String VERTX_APPLICATION = "io.vertx.launcher.application.VertxApplication"; + + // One class found: its key ("org/acme/Foo.class"), where it was read + // from, and its bytes. + static final class Found { + final String location; + final byte[] bytes; + + Found(String location, byte[] bytes) { + this.location = location; + this.bytes = bytes; + } + } + + // root is the app's package root ("org.apache.fineract"), or null when + // the main class has no package of two parts or more: then only the dirs + // and jars taken whole count. + final String root; + // launch says how the JVM was launched, or, when failed, why the app's + // classes could not be told apart. + final String launch; + final boolean failed; + private final List dirs; + private final List wholeJars; + private final List rootJars; + private final File bootJar; + private final String bootClasses; + private final String bootLib; + + private AppClasses(String root, String launch, boolean failed, List dirs, List wholeJars, + List rootJars, File bootJar, String bootClasses, String bootLib) { + this.root = root; + this.launch = launch; + this.failed = failed; + this.dirs = dirs; + this.wholeJars = wholeJars; + this.rootJars = rootJars; + this.bootJar = bootJar; + this.bootClasses = bootClasses; + this.bootLib = bootLib; + } + + private static AppClasses failed(String why) { + List none = Collections.emptyList(); + return new AppClasses(null, why, true, none, none, none, null, null, null); + } + + /** + * derive finds how the JVM was launched from its classpath + * ({@code java.class.path}), its command ({@code sun.java.command}: the + * main class or jar, then the app's arguments) and its working dir. + */ + static AppClasses derive(String classpath, String command, String userDir) { + String first = firstToken(command); + if (first.isEmpty()) { + return failed("the JVM's command names no main class or jar"); + } + File jar = archiveFile(first, userDir); + if (jar == null) { + jar = soleArchive(classpath, command, userDir); + } + if (jar != null) { + return fromJar(jar, "java -jar " + jar.getPath()); + } + int slash = first.indexOf('/'); + if (slash > 0 && CLASS_NAME.matcher(first.substring(0, slash)).matches() + && CLASS_NAME.matcher(first.substring(slash + 1)).matches()) { + return failed("the JVM runs the module " + first + " (java -m), and the app's classes are read only" + + " from a classpath"); + } + if (!CLASS_NAME.matcher(first).matches()) { + return failed("the JVM's command starts with " + first + ", which is neither a jar nor a class"); + } + List entries = classpathEntries(classpath, userDir); + if (first.startsWith(BOOT_LAUNCHER)) { + for (File entry : entries) { + if (entry.isFile() && isArchive(entry) && isBootJar(entry)) { + return fromJar(entry, "Spring Boot's launcher on " + entry.getPath()); + } + // A jar extracted to a dir, as Spring Boot's layered Docker + // images run it. + for (String inf : new String[] {"BOOT-INF/", "WEB-INF/"}) { + if (entry.isDirectory() && new File(entry, inf + "classes").isDirectory()) { + return fromExtractedBoot(entry, inf, "Spring Boot's launcher on " + entry.getPath()); + } + } + } + return failed("Spring Boot's launcher " + first + " runs with no Spring Boot jar on the classpath"); + } + if (isVertx(first)) { + String verticle = verticle(command); + if (verticle != null) { + return fromClasspath(verticle, entries, "java -cp ... " + first + " (Vert.x) running " + verticle); + } + } + String launcher = launcher(first); + if (launcher != null) { + return failed("java -cp ... " + first + ", the launcher of " + launcher + ", not the app"); + } + return fromClasspath(first, entries, "java -cp ... " + first); + } + + // fromJar reads a jar run with java -jar (or a Spring Boot jar run by its + // launcher). + private static AppClasses fromJar(File jar, String launch) { + String[] boot; + String main; + Set groupIds = new LinkedHashSet<>(); + List classPath = new ArrayList<>(); + boolean shaded; + try (JarFile jarFile = new JarFile(jar)) { + boot = bootLayout(jarFile); + int poms = addGroupIds(jarFile, groupIds); + Manifest manifest = jarFile.getManifest(); + Attributes attributes = manifest == null ? new Attributes() : manifest.getMainAttributes(); + String mainClass = trimmed(attributes.getValue(Attributes.Name.MAIN_CLASS)); + String start = trimmed(attributes.getValue("Start-Class")); + // A Spring Boot jar's Main-Class is its launcher (or, in a thin + // jar, the launcher's wrapper, with no BOOT-INF); Start-Class is + // the app's. + boolean boots = boot != null || start != null + || (mainClass != null && mainClass.startsWith(BOOT_LAUNCHER)); + main = boots ? start : mainClass; + if (main == null) { + return failed(launch + ", but its manifest names no " + (boots ? "Start-Class" : "Main-Class")); + } + if (!boots && isVertx(main)) { + // A Vert.x fat jar: Main-Class is Vert.x's launcher, and + // Main-Verticle the app's (or "js:..." and the like, which + // is no class). + String verticle = trimmed(attributes.getValue("Main-Verticle")); + if (verticle == null || !CLASS_NAME.matcher(verticle).matches()) { + return failed(launch + ", whose main class " + main + " is Vert.x's launcher, and its" + + " manifest names no Main-Verticle class"); + } + main = verticle; + } + if (boot == null) { + String launcher = launcher(main); + if (launcher != null) { + return failed(launch + ", whose main class " + main + " is the launcher of " + launcher + + ", not the app"); + } + // A jar whose main class is in a jar its Class-Path names is + // a launcher stub (Quarkus's quarkus-run.jar, Maven's test + // fork booter): its main class's package is not the app's. + if (jarFile.getEntry(classFile(main)) == null) { + return failed(launch + ", but its main class " + main + " is not in the jar, which only" + + " launches the app"); + } + shaded = isShaded(poms, groupIds, main); + String listed = attributes.getValue(Attributes.Name.CLASS_PATH); + if (listed != null) { + classPath.addAll(Arrays.asList(listed.trim().split("\\s+"))); + } + } else { + shaded = false; + } + } catch (IOException e) { + return failed(launch + ", but the jar could not be read: " + e.getMessage()); + } + String root = root(main, groupIds); + List none = Collections.emptyList(); + if (boot != null) { + return new AppClasses(root, launch, false, none, none, none, jar, boot[0], boot[1]); + } + if (shaded && root == null) { + return failed(launch + ", a shaded jar whose main class " + main + + " has no package of two parts or more"); + } + // The jars its manifest's Class-Path names (as Spring Boot's own + // extract writes them, lib/*.jar) may hold the app's other modules. + List rootJars = new ArrayList<>(); + if (shaded) { + rootJars.add(jar); + } + for (String part : classPath) { + File listed = classPathFile(jar, part); + if (listed != null && listed.isFile() && isArchive(listed)) { + rootJars.add(listed); + } + } + return new AppClasses(root, launch, false, none, + shaded ? none : Collections.singletonList(jar), rootJars, null, null, null); + } + + // fromExtractedBoot reads a Spring Boot jar extracted to dir: its + // classes dir whole, and the classes under the root in its lib dir's jars. + private static AppClasses fromExtractedBoot(File dir, String inf, String launch) { + String main = null; + File manifestFile = new File(dir, JarFile.MANIFEST_NAME); + try (InputStream input = Files.newInputStream(manifestFile.toPath())) { + main = trimmed(new Manifest(input).getMainAttributes().getValue("Start-Class")); + } catch (IOException e) { + // Said below as a manifest with no Start-Class. + } + if (main == null) { + return failed(launch + ", but " + manifestFile.getPath() + " names no Start-Class"); + } + Set groupIds = new LinkedHashSet<>(); + File[] mavenDirs = new File(dir, MAVEN_DIR).listFiles(File::isDirectory); + if (mavenDirs != null) { + for (File groupDir : mavenDirs) { + groupIds.add(groupDir.getName()); + } + } + String root = root(main, groupIds); + List libs = new ArrayList<>(); + File[] files = new File(dir, inf + "lib").listFiles(); + if (files != null) { + for (File file : files) { + if (file.isFile() && isArchive(file)) { + libs.add(file); + } + } + Collections.sort(libs); + } + return new AppClasses(root, launch, false, Collections.singletonList(new File(dir, inf + "classes")), + Collections.emptyList(), libs, null, null, null); + } + + // fromClasspath reads a classpath run as java -cp ... main: its dirs and + // the jar that holds the main class whole (that jar only under the root + // when it is shaded), and its other jars that hold a class under the root. + private static AppClasses fromClasspath(String main, List entries, String launch) { + String classFile = classFile(main); + String cut = root(main, Collections.emptySet()); + // The root is cut, or a longer groupId under it, so a jar with no + // class under cut has none under the root either. + String cutPrefix = cut == null ? null : cut.replace('.', '/') + "/"; + Set groupIds = new LinkedHashSet<>(); + List dirs = new ArrayList<>(); + List wholeJars = new ArrayList<>(); + List rootJars = new ArrayList<>(); + boolean home = false; + for (File entry : entries) { + if (entry.isDirectory()) { + dirs.add(entry); + home |= new File(entry, classFile).isFile(); + } else if (entry.isFile() && isArchive(entry)) { + try (JarFile jarFile = new JarFile(entry)) { + Set own = new LinkedHashSet<>(); + int poms = addGroupIds(jarFile, own); + groupIds.addAll(own); + // The first entry that holds the main class is where the + // JVM loads it from: the app's own jar. + if (!home && jarFile.getEntry(classFile) != null) { + home = true; + if (!isShaded(poms, own, main)) { + wholeJars.add(entry); + continue; + } + } + if (cutPrefix != null && hasClassUnder(jarFile, cutPrefix)) { + rootJars.add(entry); + } + } catch (IOException e) { + // A jar the JVM cannot read either holds none of the + // app's classes. + } + } + } + if (!home) { + return failed(launch + ", but no dir or jar on the classpath holds " + classFile); + } + String root = root(main, groupIds); + if (root == null && dirs.isEmpty() && wholeJars.isEmpty()) { + return failed(launch + ", from a shaded jar, and the main class has no package of two parts or more"); + } + return new AppClasses(root, launch, false, dirs, wholeJars, rootJars, null, null, null); + } + + /** + * root is the app's package root: the main class's package cut to its + * first three parts ({@code org.apache.fineract} for + * {@code org.apache.fineract.ServerApplication}), or a longer Maven + * groupId that the package is in, so that {@code com.acme.shop.orders} + * stays apart from a sibling {@code com.acme.shop.billing} library. A + * shorter groupId never widens it: {@code org.springframework} must not + * make Spring's classes the app's for + * {@code org.springframework.samples.petclinic}. Null when the package + * has fewer than two parts, as one part ({@code com}) is no app's own. + */ + static String root(String mainClass, Collection groupIds) { + int lastDot = mainClass.lastIndexOf('.'); + if (lastDot <= 0) { + return null; + } + String pkg = mainClass.substring(0, lastDot); + String[] parts = pkg.split("\\."); + if (parts.length < 2) { + return null; + } + String root = cut(parts); + for (String groupId : groupIds) { + if (groupId.length() > root.length() && (pkg.equals(groupId) || pkg.startsWith(groupId + "."))) { + root = groupId; + } + } + return root; + } + + private static String cut(String[] parts) { + StringBuilder cut = new StringBuilder(parts[0]); + for (int i = 1; i < Math.min(ROOT_PARTS, parts.length); i++) { + cut.append('.').append(parts[i]); + } + return cut.toString(); + } + + /** + * isShaded says whether a jar that holds the main class holds other + * projects' code too: maven-shade, the assembly plugin and Gradle's + * shadow copy each dependency's {@code pom.properties}, so a shaded jar + * has more than one, and some of another project. A groupId is the app's + * when it and the main class's package cut to three parts are one in the + * other ({@code com.example} for {@code com.example.demo}). A jar shaded + * of dependencies that carry no {@code pom.properties} (built by Gradle) + * reads as not shaded, and is taken whole, as it was before. + */ + static boolean isShaded(int poms, Collection groupIds, String mainClass) { + if (poms < 2) { + return false; + } + int lastDot = mainClass.lastIndexOf('.'); + String app = lastDot <= 0 ? "" : cut(mainClass.substring(0, lastDot).split("\\.")); + for (String groupId : groupIds) { + boolean own = !app.isEmpty() && (app.equals(groupId) || app.startsWith(groupId + ".") + || groupId.startsWith(app + ".")); + if (!own) { + return true; + } + } + return false; + } + + // launcher names what a main class launches when it is a known + // launcher's, or null. + static String launcher(String mainClass) { + for (String name : LAUNCHERS) { + if (name.endsWith(".") ? mainClass.startsWith(name) : mainClass.equals(name)) { + return name.endsWith(".") ? name.substring(0, name.length() - 1) : name; + } + } + return null; + } + + private static boolean isVertx(String mainClass) { + return VERTX_LAUNCHER.equals(mainClass) || VERTX_APPLICATION.equals(mainClass); + } + + // verticle is the class a Vert.x launch command runs: "Launcher run + // com.acme.Main ..." (Vert.x 4) or "VertxApplication com.acme.Main ..." + // (Vert.x 5), or null when it names none. + static String verticle(String command) { + String[] words = command.trim().split("\\s+"); + int at = 1; + if (at < words.length && words[at].equals("run")) { + at++; + } + if (at >= words.length || words[at].indexOf('.') < 0 || !CLASS_NAME.matcher(words[at]).matches()) { + return null; + } + return words[at]; + } + + /** + * scan puts every app class it finds in out, keyed "org/acme/Foo.class", + * unless skip takes the key or out has one by that key already: the first + * one found wins, as with the JVM's class loading. + */ + void scan(Map out, Predicate skip) { + if (failed) { + return; + } + String prefix = root == null ? null : root.replace('.', '/') + "/"; + for (File dir : dirs) { + File from = walkFrom(dir, prefix); + if (from != null) { + scanDirectory(from, dir, out, skip); + } + } + for (File jar : wholeJars) { + scanJar(jar, "", out, skip); + } + // The root-filtered jars give nothing when there is no root. + if (prefix != null) { + for (File jar : rootJars) { + scanJar(jar, prefix, out, skip); + } + } + if (bootJar != null) { + scanBootJar(prefix, out, skip); + } + } + + // walkFrom is where a classpath dir is read from: the whole dir, but for + // the file system's root ("java Main" run in /, whose classpath is "."), + // only the package root's own dir, as a walk of / never ends in time; + // null there when there is no root. + static File walkFrom(File dir, String prefix) { + if (dir.getAbsoluteFile().toPath().getParent() != null) { + return dir; + } + return prefix == null ? null : new File(dir, prefix); + } + + /** + * walkClassFiles calls each with every .class file under start. A file + * or dir it cannot read is passed over, as the JVM cannot load from it + * either, and never ends the walk: Files.walk throws for the first one, + * which left the whole index empty. + */ + static void walkClassFiles(Path start, Consumer each) throws IOException { + Files.walkFileTree(start, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attributes) { + // Without FOLLOW_LINKS the attributes are a link's own, so a + // link to a class file (a symlink farm, a projected volume) + // is followed here, as the JVM follows it. + boolean regular = attributes.isRegularFile() + || (attributes.isSymbolicLink() && Files.isRegularFile(file)); + if (regular && file.toString().endsWith(".class")) { + each.accept(file); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e) { + return FileVisitResult.CONTINUE; + } + }); + } + + private static void scanDirectory(File walkFrom, File base, Map out, Predicate skip) { + if (!walkFrom.isDirectory()) { + return; + } + Path basePath = base.toPath(); + try { + walkClassFiles(walkFrom.toPath(), path -> { + String key = basePath.relativize(path).toString().replace(File.separatorChar, '/'); + if (skip.test(key) || out.containsKey(key)) { + return; + } + try { + out.put(key, new Found(path.toAbsolutePath().toString() + .replace(File.separatorChar, '/'), Files.readAllBytes(path))); + } catch (IOException e) { + // Unreadable: the JVM cannot load it either. + } + }); + } catch (IOException e) { + // A directory gone or unreadable holds none of the app's classes. + } + } + + // scanJar reads the classes under prefix in a jar ("" for all of it, but + // for a Spring Boot launcher's own classes, never the app's). + private static void scanJar(File jar, String prefix, Map out, Predicate skip) { + try (JarFile jarFile = new JarFile(jar)) { + Enumeration entries = jarFile.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + String name = entry.getName(); + if (!name.startsWith(prefix) || !name.endsWith(".class") || name.startsWith(BOOT_LAUNCHER_PATH) + || skip.test(name) || out.containsKey(name)) { + continue; + } + try (InputStream input = jarFile.getInputStream(entry)) { + out.put(name, new Found(jar.getAbsolutePath() + "!" + name, readAllBytes(input))); + } + } + } catch (IOException e) { + // As for a directory: an unreadable jar holds none of the app's classes. + } + } + + // scanBootJar reads a Spring Boot jar: its classes dir whole first, as + // its launcher puts it first on the classpath, then the classes under + // the root in its lib dir's jars (none when there is no root). + private void scanBootJar(String prefix, Map out, Predicate skip) { + String location = bootJar.getAbsolutePath(); + try (JarFile jarFile = new JarFile(bootJar)) { + List libs = new ArrayList<>(); + Enumeration entries = jarFile.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + String name = entry.getName(); + if (name.startsWith(bootLib) && name.endsWith(".jar")) { + libs.add(entry); + continue; + } + if (!name.startsWith(bootClasses) || !name.endsWith(".class")) { + continue; + } + String key = name.substring(bootClasses.length()); + if (skip.test(key) || out.containsKey(key)) { + continue; + } + try (InputStream input = jarFile.getInputStream(entry)) { + out.put(key, new Found(location + "!" + name, readAllBytes(input))); + } + } + if (prefix == null || libs.isEmpty()) { + return; + } + // Spring Boot stores its libs uncompressed, so each is read in + // place from the jar mapped read-only, off the app's heap: a lib + // copied whole into the heap (an AWS SDK bundle, a native-library + // jar: hundreds of MB) could run the app out of memory. A lib + // read this way fails over to a stream, whose heap use is one + // class at a time. + ByteBuffer mapped = map(bootJar); + Map stored = new HashMap<>(); + List items = mapped == null ? null : centralDirectory(mapped, 0, mapped.limit()); + if (items != null) { + for (ZipItem item : items) { + if (item.method == 0 && item.compressed == item.size) { + stored.put(item.name, item); + } + } + } + for (JarEntry lib : libs) { + String libLocation = location + "!" + lib.getName() + "!"; + ZipItem item = stored.get(lib.getName()); + long data = item == null ? -1 : dataOffset(mapped, 0, mapped.limit(), item); + if (data >= 0 && data + item.size <= mapped.limit() + && scanZip(mapped, (int) data, (int) item.size, prefix, libLocation, out, skip)) { + continue; + } + try (InputStream input = jarFile.getInputStream(lib)) { + scanJarStream(input, prefix, libLocation, out, skip); + } catch (IOException e) { + // One unreadable nested jar does not hide the others. + } + } + } catch (IOException e) { + // As for a plain jar. + } + } + + // map is a file mapped read-only, or null when it cannot be (over 2 GB, + // or a file system that does not map). + private static ByteBuffer map(File file) { + try (FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ)) { + long size = channel.size(); + return size > Integer.MAX_VALUE ? null : channel.map(FileChannel.MapMode.READ_ONLY, 0, size); + } catch (IOException | UnsupportedOperationException e) { + return null; + } + } + + private static void scanJarStream(InputStream input, String prefix, String location, Map out, + Predicate skip) throws IOException { + try (JarInputStream nested = new JarInputStream(input)) { + JarEntry entry; + while ((entry = nested.getNextJarEntry()) != null) { + String name = entry.getName(); + if (!name.startsWith(prefix) || !name.endsWith(".class") || skip.test(name) + || out.containsKey(name)) { + continue; + } + out.put(name, new Found(location + name, readAllBytes(nested))); + } + } + } + + // One entry of a zip's central directory; local is its local header's + // offset from the zip's first byte. + static final class ZipItem { + final String name; + final int method; + final long compressed; + final long size; + final long local; + + ZipItem(String name, int method, long compressed, long size, long local) { + this.name = name; + this.method = method; + this.compressed = compressed; + this.size = size; + this.local = local; + } + } + + // A class file is far smaller; an entry that says it is larger is read + // by the stream, which reads what is there instead. + private static final long MAX_CLASS_SIZE = 64L << 20; + + /** + * scanZipBytes is scanZip over a jar held in a byte array. + */ + static boolean scanZipBytes(byte[] zip, String prefix, String location, Map out, + Predicate skip) { + return scanZip(ByteBuffer.wrap(zip), 0, zip.length, prefix, location, out, skip); + } + + /** + * scanZip reads the classes under prefix from the jar at zip[base, + * base + length) by its central directory, so that it inflates only + * those: a JarInputStream must inflate every entry to step over it, and + * a Spring Boot jar's libs are most of its size (1.25 s for 70 MB of + * libs, inside the first START). False when it is not a zip read this + * way (zip64, another compression, or a directory that disagrees with + * itself or with the data), for the caller to stream it instead; out may + * then hold some of the classes already, each read whole and right. + */ + static boolean scanZip(ByteBuffer zip, int base, int length, String prefix, String location, + Map out, Predicate skip) { + try { + List items = centralDirectory(zip, base, length); + if (items == null) { + return false; + } + for (ZipItem item : items) { + String name = item.name; + if (!name.startsWith(prefix) || !name.endsWith(".class") || skip.test(name) + || out.containsKey(name)) { + continue; + } + byte[] bytes = entryBytes(zip, base, length, item); + if (bytes == null) { + return false; + } + out.put(name, new Found(location + name, bytes)); + } + return true; + } catch (IndexOutOfBoundsException | BufferUnderflowException | IllegalArgumentException e) { + // Every read is bounds-checked first; this is the last guard, as + // this runs inside the app's JVM. + return false; + } + } + + /** + * centralDirectory lists the zip at zip[base, base + length), or is null + * when it is not one read this way: zip64, damaged, or an end record + * whose entry count or directory size disagrees with the directory (a + * jar of over 65535 entries written without zip64 keeps its count mod + * 65536). Data before the zip (a launch script, as Spring Boot's fully + * executable jars have) is allowed for, as the JDK's ZipFile does. + */ + static List centralDirectory(ByteBuffer zip, int base, int length) { + int end = -1; + for (int at = base + length - 22; at >= base && at >= base + length - 22 - 0xFFFF; at--) { + if (u32(zip, at) == 0x06054b50L) { + end = at; + break; + } + } + if (end < 0 || (end - 20 >= base && u32(zip, end - 20) == 0x07064b50L)) { + return null; + } + int count = u16(zip, end + 10); + long size = u32(zip, end + 12); + long directory = u32(zip, end + 16); + if (count == 0xFFFF || size > end - base) { + return null; + } + // The directory ends at the end record; its offset there is from the + // zip's first byte, so the difference is the data before the zip. + int at = (int) (end - size); + long before = at - base - directory; + if (before < 0) { + return null; + } + List items = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + if (at + 46 > end || u32(zip, at) != 0x02014b50L) { + return null; + } + int nameLength = u16(zip, at + 28); + int next = at + 46 + nameLength + u16(zip, at + 30) + u16(zip, at + 32); + if (next > end) { + return null; + } + String name = new String(copy(zip, at + 46, nameLength), StandardCharsets.UTF_8); + items.add(new ZipItem(name, u16(zip, at + 10), u32(zip, at + 20), u32(zip, at + 24), + u32(zip, at + 42) + before)); + at = next; + } + return at == end ? items : null; + } + + // dataOffset is where an entry's data starts, from base, or -1 when its + // local header is not there. + static long dataOffset(ByteBuffer zip, int base, int length, ZipItem item) { + if (item.local + 30 > length || u32(zip, base + (int) item.local) != 0x04034b50L) { + return -1; + } + int at = base + (int) item.local; + return item.local + 30 + u16(zip, at + 26) + u16(zip, at + 28); + } + + // entryBytes is one entry's bytes, or null when they cannot be read this + // way, or the data is not the size the directory says. + private static byte[] entryBytes(ByteBuffer zip, int base, int length, ZipItem item) { + long data = dataOffset(zip, base, length, item); + if (data < 0 || item.size > MAX_CLASS_SIZE || data + item.compressed > length) { + return null; + } + if (item.method == 0) { + return item.compressed == item.size ? copy(zip, base + (int) data, (int) item.size) : null; + } + if (item.method != 8) { + return null; + } + Inflater inflater = new Inflater(true); + try { + inflater.setInput(copy(zip, base + (int) data, (int) item.compressed)); + byte[] bytes = new byte[(int) item.size]; + byte[] past = new byte[1]; + int read = 0; + boolean padded = false; + while (!inflater.finished()) { + int n = read < bytes.length + ? inflater.inflate(bytes, read, bytes.length - read) + : inflater.inflate(past); + if (n > 0) { + if (read == bytes.length) { + // More data than the directory's size. + return null; + } + read += n; + } else if (inflater.needsInput() && !padded) { + // zlib may ask for one byte past a raw deflate stream. + inflater.setInput(new byte[1]); + padded = true; + } else { + return null; + } + } + return read == bytes.length ? bytes : null; + } catch (DataFormatException e) { + return null; + } finally { + inflater.end(); + } + } + + private static byte[] copy(ByteBuffer zip, int at, int length) { + ByteBuffer from = zip.duplicate(); + // Through Buffer: ByteBuffer.position(int) is Java 9's, and this + // runs on Java 8. + ((Buffer) from).position(at); + byte[] bytes = new byte[length]; + from.get(bytes); + return bytes; + } + + private static int u16(ByteBuffer zip, int at) { + return (zip.get(at) & 0xff) | (zip.get(at + 1) & 0xff) << 8; + } + + private static long u32(ByteBuffer zip, int at) { + return u16(zip, at) | (long) u16(zip, at + 2) << 16; + } + + // isBootJar says whether a jar on the classpath is a Spring Boot jar its + // launcher runs: one with BOOT-INF or WEB-INF classes, or a Start-Class. + private static boolean isBootJar(File jar) { + try (JarFile jarFile = new JarFile(jar)) { + Manifest manifest = jarFile.getManifest(); + return bootLayout(jarFile) != null + || (manifest != null && trimmed(manifest.getMainAttributes().getValue("Start-Class")) != null); + } catch (IOException e) { + return false; + } + } + + // bootLayout is {classes dir, lib dir} of a Spring Boot jar or war, or + // null for any other jar. + private static String[] bootLayout(JarFile jarFile) { + if (hasClassUnder(jarFile, "BOOT-INF/classes/")) { + return new String[] {"BOOT-INF/classes/", "BOOT-INF/lib/"}; + } + if (hasClassUnder(jarFile, "WEB-INF/classes/")) { + return new String[] {"WEB-INF/classes/", "WEB-INF/lib/"}; + } + return null; + } + + private static boolean hasClassUnder(JarFile jarFile, String prefix) { + Enumeration entries = jarFile.entries(); + while (entries.hasMoreElements()) { + String name = entries.nextElement().getName(); + if (name.startsWith(prefix) && name.endsWith(".class")) { + return true; + } + } + return false; + } + + // addGroupIds adds the groupId of each + // META-INF/maven///pom.properties, and returns how + // many there are. + private static int addGroupIds(JarFile jarFile, Set groupIds) { + int poms = 0; + Enumeration entries = jarFile.entries(); + while (entries.hasMoreElements()) { + String name = entries.nextElement().getName(); + if (!name.startsWith(MAVEN_DIR) || !name.endsWith("/pom.properties")) { + continue; + } + String[] parts = name.substring(MAVEN_DIR.length()).split("/"); + if (parts.length == 3 && !parts[0].isEmpty()) { + groupIds.add(parts[0]); + poms++; + } + } + return poms; + } + + // classPathFile is the file a manifest Class-Path entry names: a URL + // relative to the jar's dir ("lib/a%20b.jar"), or null when it is not + // a local file. + static File classPathFile(File jar, String entry) { + if (entry.isEmpty()) { + return null; + } + try { + URI uri = jar.getAbsoluteFile().getParentFile().toURI().resolve(entry); + return "file".equals(uri.getScheme()) ? new File(uri) : null; + } catch (IllegalArgumentException e) { + return null; + } + } + + private static List classpathEntries(String classpath, String userDir) { + List entries = new ArrayList<>(); + if (classpath == null) { + return entries; + } + for (String part : classpath.split(Pattern.quote(File.pathSeparator))) { + if (!part.trim().isEmpty()) { + entries.add(absolute(part.trim(), userDir)); + } + } + return entries; + } + + private static File absolute(String path, String userDir) { + File file = new File(path); + return file.isAbsolute() || userDir == null ? file : new File(userDir, path); + } + + // soleArchive is the jar run with java -jar when its path has a space + // and the command does not quote it: the classpath is then that jar + // alone, and the command starts with its path. + private static File soleArchive(String classpath, String command, String userDir) { + if (classpath == null || classpath.trim().isEmpty() || classpath.contains(File.pathSeparator) + || !command.trim().startsWith(classpath.trim())) { + return null; + } + return archiveFile(classpath, userDir); + } + + // archiveFile is the jar a command starts with, or null when it starts + // with something else (a class). + static File archiveFile(String path, String userDir) { + if (path == null || path.trim().isEmpty()) { + return null; + } + File file = absolute(path.trim(), userDir); + return file.isFile() && isArchive(file) ? file : null; + } + + static boolean isArchive(File file) { + String name = file.getName().toLowerCase(Locale.ROOT); + return name.endsWith(".jar") + || name.endsWith(".war") + || name.endsWith(".ear") + || name.endsWith(".zip"); + } + + private static String classFile(String className) { + return className.replace('.', '/') + ".class"; + } + + private static String trimmed(String value) { + return value == null || value.trim().isEmpty() ? null : value.trim(); + } + + // firstToken is a command's first word, which may be quoted. + static String firstToken(String command) { + if (command == null) { + return ""; + } + String trimmed = command.trim(); + if (trimmed.isEmpty()) { + return ""; + } + char first = trimmed.charAt(0); + if (first == '"' || first == '\'') { + int end = trimmed.indexOf(first, 1); + return end > 0 ? trimmed.substring(1, end) : trimmed.substring(1); + } + int end = 0; + while (end < trimmed.length() && !Character.isWhitespace(trimmed.charAt(end))) { + end++; + } + return trimmed.substring(0, end); + } + + private static byte[] readAllBytes(InputStream input) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(16 * 1024); + byte[] buffer = new byte[8192]; + int read; + while ((read = input.read(buffer)) != -1) { + output.write(buffer, 0, read); + } + return output.toByteArray(); + } +} diff --git a/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java b/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java index 035e3fa2..c171184a 100644 --- a/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java +++ b/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java @@ -38,7 +38,6 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -50,7 +49,6 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; -import java.util.stream.Stream; /** * Collects per-testcase JaCoCo coverage and streams the executed probe set per @@ -940,6 +938,15 @@ private void warmup() { if (isWarmupDisabled() || !warmed.compareAndSet(false, true)) { return; } + // Only the app's own classes: a library's static initializer can + // start threads, open connections or self-attach an agent (Mockito + // through byte-buddy), and the whole classpath of a large app took + // the warmup to 54 s (105k classes on Fineract). + if (!coverageIndex.isAppOnly()) { + log(Level.INFO, "Keploy dedup: no warmup, as the index is not the app's own classes alone" + + " (from " + coverageIndex.source + ")", null); + return; + } ClassLoader[] loaders = warmupLoaders(); int initialized = 0; int failed = 0; @@ -1374,6 +1381,20 @@ private List entries() { } } + // Whether the index holds only the app's own classes (named in + // KEPLOY_JAVA_CLASS_DIRS, or derived from the launch), and where it + // came from. Set by loadEntries, before entries() returns. + private volatile boolean appOnly; + private volatile String source = ""; + + // isAppOnly says whether the index holds only the app's own classes. + // When it does not (a whole jar or the whole classpath), fingerprints + // and the warmup take in library code too. + private boolean isAppOnly() { + entries(); + return appOnly; + } + private List loadEntries() { LinkedHashMap collected = new LinkedHashMap<>(); List applicationRoots = applicationRoots(); @@ -1381,21 +1402,68 @@ private List loadEntries() { List classpathRoots = Collections.emptyList(); scanRoots(applicationRoots, collected); + List foundRoots = new ArrayList<>(); + for (File root : applicationRoots) { + if (root.exists()) { + foundRoots.add(root); + } + } + String from = "the class dirs " + summarizeFiles(foundRoots, 5); + boolean onlyApp = true; + // why says why the index is not the app's own classes alone. + String why = ""; if (collected.isEmpty()) { + AppClasses derived = AppClasses.derive(System.getProperty("java.class.path", ""), + System.getProperty("sun.java.command", ""), System.getProperty("user.dir")); + Map found = new LinkedHashMap<>(); + derived.scan(found, this::shouldSkipClass); + for (Map.Entry entry : found.entrySet()) { + collected.put(entry.getKey(), new ClassEntry(classNameFromKey(entry.getKey()), + normalizePath(entry.getValue().location), entry.getValue().bytes)); + } + from = derived.root == null ? derived.launch + : "the package " + derived.root + " (" + derived.launch + ")"; + why = derived.failed ? derived.launch : "no class found from " + from; + } + if (collected.isEmpty()) { + onlyApp = false; executableArchiveRoots = executableArchiveRoots(); scanRoots(executableArchiveRoots, collected); + from = "the whole jar " + summarizeFiles(executableArchiveRoots, 5); } if (collected.isEmpty() && isClasspathFallbackEnabled()) { classpathRoots = classpathRoots(); scanRoots(classpathRoots, collected); + from = "the whole classpath"; } - if (collected.isEmpty() && diagnosticsEnabled()) { - diagnostic("no application classes indexed" - + ", applicationRoots=" + summarizeFiles(applicationRoots, 5) - + ", executableArchiveRoots=" + summarizeFiles(executableArchiveRoots, 5) - + ", classpathFallbackRoots=" + summarizeFiles(classpathRoots, 5) - + ", java.class.path=" + System.getProperty("java.class.path", "") - + ", sun.java.command=" + System.getProperty("sun.java.command", "")); + appOnly = onlyApp; + source = collected.isEmpty() ? "no class" : from; + if (collected.isEmpty()) { + // Said at WARNING: no test's coverage is measured at all. + log(Level.WARNING, "Keploy dedup found none of the app's classes (" + why + "), nor any class" + + (isClasspathFallbackEnabled() ? " in the jar or on the classpath" + : " in the jar, and KEPLOY_JAVA_CLASSPATH_FALLBACK=false keeps it off the classpath") + + ", so no test's coverage is measured. Set KEPLOY_JAVA_CLASS_DIRS to the app's class" + + " dirs and jars", null); + if (diagnosticsEnabled()) { + diagnostic("no application classes indexed" + + ", applicationRoots=" + summarizeFiles(applicationRoots, 5) + + ", executableArchiveRoots=" + summarizeFiles(executableArchiveRoots, 5) + + ", classpathFallbackRoots=" + summarizeFiles(classpathRoots, 5) + + ", java.class.path=" + System.getProperty("java.class.path", "") + + ", sun.java.command=" + System.getProperty("sun.java.command", "")); + } + } else if (onlyApp) { + log(Level.INFO, "Keploy dedup: fingerprinting the app's " + collected.size() + + " classes, from " + from + ". Set KEPLOY_JAVA_CLASS_DIRS to the app's class dirs" + + " and jars if that is not the app's own code", null); + } else { + // Said at WARNING: dedup still runs, but on library code too. + log(Level.WARNING, "Keploy dedup could not tell the app's own classes from its libraries (" + + why + "), so it fingerprints all " + collected.size() + " classes of " + from + + ": library code makes duplicate tests look different, so dedup keeps more tests" + + " than it should, and the startup warmup is skipped. Set KEPLOY_JAVA_CLASS_DIRS" + + " to the app's class dirs and jars", null); } List sorted = new ArrayList<>(collected.values()); @@ -1444,7 +1512,7 @@ private boolean isClasspathFallbackEnabled() { private List executableArchiveRoots() { LinkedHashSet roots = new LinkedHashSet<>(); - addArchiveRoot(roots, firstCommandToken(System.getProperty("sun.java.command", ""))); + addArchiveRoot(roots, AppClasses.firstToken(System.getProperty("sun.java.command", ""))); String classpath = System.getProperty("java.class.path", ""); if (classpath.trim().isEmpty()) { @@ -1472,42 +1540,11 @@ private void addArchiveRoot(Set roots, String rawPath) { if (!file.isAbsolute()) { file = new File(System.getProperty("user.dir"), path); } - if (file.isFile() && isArchive(file)) { + if (file.isFile() && AppClasses.isArchive(file)) { roots.add(file); } } - private boolean isArchive(File file) { - String name = file.getName().toLowerCase(Locale.ROOT); - return name.endsWith(".jar") - || name.endsWith(".war") - || name.endsWith(".ear") - || name.endsWith(".zip"); - } - - private String firstCommandToken(String command) { - if (command == null) { - return ""; - } - - String trimmed = command.trim(); - if (trimmed.isEmpty()) { - return ""; - } - - char first = trimmed.charAt(0); - if (first == '"' || first == '\'') { - int end = trimmed.indexOf(first, 1); - return end > 0 ? trimmed.substring(1, end) : trimmed.substring(1); - } - - int end = 0; - while (end < trimmed.length() && !Character.isWhitespace(trimmed.charAt(end))) { - end++; - } - return trimmed.substring(0, end); - } - private String[] configuredRoots(String configured) { if (configured.indexOf(',') >= 0) { return configured.split(","); @@ -1523,7 +1560,7 @@ private List classpathRoots() { for (String part : parts) { if (!part.trim().isEmpty()) { File file = new File(part.trim()); - if (file.isDirectory() || isArchive(file)) { + if (file.isDirectory() || AppClasses.isArchive(file)) { roots.add(file); } } @@ -1539,7 +1576,7 @@ private void scanRoots(List roots, Map output) { } if (root.isDirectory()) { scanDirectory(root, output); - } else if (root.isFile() && isArchive(root)) { + } else if (root.isFile() && AppClasses.isArchive(root)) { scanJar(root, output); } } @@ -1547,10 +1584,8 @@ private void scanRoots(List roots, Map output) { private void scanDirectory(File root, Map output) { Path base = root.toPath(); - try (Stream stream = Files.walk(base)) { - stream.filter(Files::isRegularFile) - .filter(path -> path.toString().endsWith(".class")) - .forEach(path -> addClassFile(base, path, output)); + try { + AppClasses.walkClassFiles(base, path -> addClassFile(base, path, output)); } catch (IOException e) { log(Level.FINE, "Failed to scan class directory " + root.getAbsolutePath(), e); } diff --git a/keploy-sdk/src/test/java/io/keploy/dedup/AppClassesTest.java b/keploy-sdk/src/test/java/io/keploy/dedup/AppClassesTest.java new file mode 100644 index 00000000..e35a977e --- /dev/null +++ b/keploy-sdk/src/test/java/io/keploy/dedup/AppClassesTest.java @@ -0,0 +1,834 @@ +package io.keploy.dedup; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.jar.Attributes; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import java.util.zip.CRC32; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.junit.Assume; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +// AppClassesTest checks which classes count as the app's own for each way a +// JVM is launched: a classpath (Jib), a Spring Boot jar, a shaded jar. +public class AppClassesTest { + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + @Test + public void rootIsTheMainPackageCutToThreePartsOrALongerGroupId() { + assertEquals("org.apache.fineract", + AppClasses.root("org.apache.fineract.ServerApplication", Collections.emptySet())); + assertEquals("com.acme", AppClasses.root("com.acme.Main", Collections.emptySet())); + assertEquals("com.acme.shop", AppClasses.root("com.acme.shop.web.boot.Main", Collections.emptySet())); + // A longer groupId the package is in narrows the root. + assertEquals("com.acme.shop.orders", AppClasses.root("com.acme.shop.orders.app.Main", + Arrays.asList("com.acme.shop.billing", "com.acme.shop.orders", "com.acme.shop.orders.app.x"))); + // The groupId equal to the whole package counts too. + assertEquals("com.acme.shop.orders", AppClasses.root("com.acme.shop.orders.Main", + Collections.singletonList("com.acme.shop.orders"))); + // A shorter groupId never widens it: Spring's own classes are not + // PetClinic's. + assertEquals("org.springframework.samples", AppClasses.root( + "org.springframework.samples.petclinic.PetClinicApplication", + Collections.singletonList("org.springframework"))); + // A groupId that only shares a prefix of a part is not the package's. + assertEquals("com.acme.shop", AppClasses.root("com.acme.shop.orders.Main", + Collections.singletonList("com.acme.shop.ord"))); + // One part, or none, is no app's own. + assertNull(AppClasses.root("Main", Collections.emptySet())); + assertNull(AppClasses.root("app.Main", Collections.emptySet())); + } + + @Test + public void classpathLaunchTakesItsDirsAndTheRootsClassesInItsJars() throws IOException { + File classes = tmp.newFolder("app", "classes"); + write(new File(classes, "org/acme/shop/Main.class")); + write(new File(classes, "org/acme/shop/web/Controller.class")); + // A class outside the root in the app's own dir is still the app's. + write(new File(classes, "org/acme/generated/Mapper.class")); + File libs = tmp.newFolder("app", "libs"); + File core = jar(new File(libs, "shop-core.jar"), null, + "org/acme/shop/core/Service.class", "META-INF/maven/org.acme.shop/shop-core/pom.properties", + // The dir's copy wins, as on the JVM's classpath. + "org/acme/shop/Main.class"); + File spring = jar(new File(libs, "spring-web.jar"), null, + "org/springframework/web/Servlet.class", "META-INF/maven/org.springframework/spring-web/pom.properties"); + File commons = jar(new File(libs, "acme-commons.jar"), null, "org/acme/commons/Util.class"); + String classpath = join(classes, core, spring, commons, new File(libs, "missing.jar")); + + AppClasses app = AppClasses.derive(classpath, "org.acme.shop.Main --port 8080", tmp.getRoot().getPath()); + + assertEquals("org.acme.shop", app.root); + Map found = scan(app); + assertEquals(new TreeSet<>(Arrays.asList("org/acme/shop/Main.class", "org/acme/shop/web/Controller.class", + "org/acme/generated/Mapper.class", "org/acme/shop/core/Service.class")), found.keySet()); + assertTrue(found.get("org/acme/shop/Main.class").location, + found.get("org/acme/shop/Main.class").location.startsWith(classes.getAbsolutePath())); + assertEquals("org/acme/shop/core/Service.class", + new String(found.get("org/acme/shop/core/Service.class").bytes, StandardCharsets.UTF_8)); + } + + @Test + public void classpathLaunchNarrowsToALongerGroupId() throws IOException { + File orders = jar(tmp.newFile("orders.jar"), null, "com/acme/shop/orders/Main.class", + "com/acme/shop/orders/Order.class", "META-INF/maven/com.acme.shop.orders/orders/pom.properties"); + File billing = jar(tmp.newFile("billing.jar"), null, "com/acme/shop/billing/Invoice.class", + "META-INF/maven/com.acme.shop.billing/billing/pom.properties"); + + AppClasses app = AppClasses.derive(join(orders, billing), "com.acme.shop.orders.Main", null); + + assertEquals("com.acme.shop.orders", app.root); + assertEquals(new TreeSet<>(Arrays.asList("com/acme/shop/orders/Main.class", + "com/acme/shop/orders/Order.class")), scan(app).keySet()); + } + + @Test + public void skipLeavesOutWhatTheIndexNeverTakes() throws IOException { + File classes = tmp.newFolder("classes"); + write(new File(classes, "org/acme/shop/Main.class")); + write(new File(classes, "org/acme/shop/MainTest.class")); + File lib = jar(tmp.newFile("lib.jar"), null, "org/acme/shop/LibTest.class", "org/acme/shop/Lib.class"); + + Map found = new LinkedHashMap<>(); + AppClasses.derive(join(classes, lib), "org.acme.shop.Main", null) + .scan(found, key -> key.endsWith("Test.class")); + + assertEquals(new TreeSet<>(Arrays.asList("org/acme/shop/Main.class", "org/acme/shop/Lib.class")), + new TreeSet<>(found.keySet())); + } + + @Test + public void springBootJarTakesItsClassesAndTheRootsClassesInItsLibs() throws IOException { + File app = springBootJar("BOOT-INF/"); + + AppClasses fromJar = AppClasses.derive(app.getName(), app.getName() + " --server.port=8080", + app.getParent()); + assertEquals("com.example.demo", fromJar.root); + Map found = scan(fromJar); + assertEquals(new TreeSet<>(Arrays.asList("com/example/demo/DemoApplication.class", + "com/example/other/Helper.class", "com/example/demo/model/Item.class")), found.keySet()); + // The app's own copy wins over a lib's. + assertTrue(found.get("com/example/demo/DemoApplication.class").location, + found.get("com/example/demo/DemoApplication.class").location + .endsWith("!BOOT-INF/classes/com/example/demo/DemoApplication.class")); + assertTrue(found.get("com/example/demo/model/Item.class").location, + found.get("com/example/demo/model/Item.class").location + .endsWith("!BOOT-INF/lib/demo-model.jar!com/example/demo/model/Item.class")); + + // skip applies to its classes and its libs' alike. + Map skipped = new LinkedHashMap<>(); + fromJar.scan(skipped, key -> key.endsWith("Helper.class") || key.endsWith("Item.class")); + assertEquals(Collections.singleton("com/example/demo/DemoApplication.class"), skipped.keySet()); + + // The same jar run by Spring Boot's launcher on -cp. + AppClasses fromLauncher = AppClasses.derive(app.getPath(), + "org.springframework.boot.loader.launch.JarLauncher", null); + assertEquals("com.example.demo", fromLauncher.root); + assertEquals(found.keySet(), scan(fromLauncher).keySet()); + } + + @Test + public void springBootWarUsesWebInf() throws IOException { + File war = springBootJar("WEB-INF/"); + + AppClasses app = AppClasses.derive(war.getPath(), war.getPath(), null); + + assertEquals("com.example.demo", app.root); + assertEquals(new TreeSet<>(Arrays.asList("com/example/demo/DemoApplication.class", + "com/example/other/Helper.class", "com/example/demo/model/Item.class")), scan(app).keySet()); + } + + @Test + public void extractedSpringBootDirIsReadLikeItsJar() throws IOException { + File dir = tmp.newFolder("application"); + write(new File(dir, "BOOT-INF/classes/com/example/demo/DemoApplication.class")); + write(new File(dir, "BOOT-INF/classes/com/example/other/Helper.class")); + jar(new File(dir, "BOOT-INF/lib/demo-model.jar"), null, "com/example/demo/model/Item.class"); + jar(new File(dir, "BOOT-INF/lib/jackson.jar"), null, "com/fasterxml/jackson/databind/ObjectMapper.class"); + String launcher = "org.springframework.boot.loader.launch.JarLauncher"; + assertFailed(AppClasses.derive(dir.getPath(), launcher, null), "names no Start-Class"); + + File manifest = new File(dir, "META-INF/MANIFEST.MF"); + manifest.getParentFile().mkdirs(); + Manifest withStart = manifest(launcher); + withStart.getMainAttributes().putValue("Start-Class", "com.example.demo.web.DemoApplication"); + try (FileOutputStream out = new FileOutputStream(manifest)) { + withStart.write(out); + } + // A longer groupId narrows the root here too: demo-model's + // com.example.demo.model is a sibling's, not under com.example.demo.web. + new File(dir, "META-INF/maven/com.example.demo.web/demo").mkdirs(); + jar(new File(dir, "BOOT-INF/lib/demo-extra.jar"), null, "com/example/demo/web/x/Extra.class"); + AppClasses app = AppClasses.derive(dir.getPath(), launcher, null); + + assertEquals("com.example.demo.web", app.root); + assertEquals(new TreeSet<>(Arrays.asList("com/example/demo/DemoApplication.class", + "com/example/other/Helper.class", "com/example/demo/web/x/Extra.class")), scan(app).keySet()); + + // Extracted from a war: WEB-INF. + File war = tmp.newFolder("war"); + write(new File(war, "WEB-INF/classes/com/example/shop/ShopApplication.class")); + File warManifest = new File(war, "META-INF/MANIFEST.MF"); + warManifest.getParentFile().mkdirs(); + Manifest warStart = manifest("org.springframework.boot.loader.launch.WarLauncher"); + warStart.getMainAttributes().putValue("Start-Class", "com.example.shop.ShopApplication"); + try (FileOutputStream out = new FileOutputStream(warManifest)) { + warStart.write(out); + } + assertEquals(Collections.singleton("com/example/shop/ShopApplication.class"), + scan(AppClasses.derive(war.getPath(), "org.springframework.boot.loader.launch.WarLauncher", null)) + .keySet()); + } + + @Test + public void plainJarAddsTheJarsItsClassPathNames() throws IOException { + File dir = tmp.newFolder("extracted"); + jar(new File(dir, "lib/shop-core.jar"), null, "com/acme/shop/core/Service.class"); + jar(new File(dir, "lib/jackson.jar"), null, "com/fasterxml/jackson/databind/ObjectMapper.class"); + File app = new File(dir, "app.jar"); + Manifest manifest = manifest("com.acme.shop.Main"); + manifest.getMainAttributes().put(Attributes.Name.CLASS_PATH, " lib/shop-core.jar lib/jackson.jar lib/gone.jar "); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(app), manifest)) { + entry(out, "com/acme/shop/Main.class", null); + } + + AppClasses fromJar = AppClasses.derive(app.getPath(), app.getPath(), null); + + assertEquals("com.acme.shop", fromJar.root); + assertEquals(new TreeSet<>(Arrays.asList("com/acme/shop/Main.class", "com/acme/shop/core/Service.class")), + scan(fromJar).keySet()); + } + + @Test + public void shadedJarTakesOnlyTheRootsClasses() throws IOException { + File shaded = jar(tmp.newFile("hello.jar"), "com.example.helloworld.HelloWorldApplication", + "com/example/helloworld/HelloWorldApplication.class", + "com/example/helloworld/resources/HelloResource.class", + "org/eclipse/jetty/server/Server.class", + "org/glassfish/jersey/server/ResourceConfig.class", + "META-INF/maven/com.example/helloworld/pom.properties", + "META-INF/maven/org.eclipse.jetty/jetty-server/pom.properties"); + + AppClasses app = AppClasses.derive(shaded.getPath(), shaded.getPath(), null); + + assertEquals("com.example.helloworld", app.root); + assertEquals(new TreeSet<>(Arrays.asList("com/example/helloworld/HelloWorldApplication.class", + "com/example/helloworld/resources/HelloResource.class")), scan(app).keySet()); + } + + @Test + public void failsWithAReasonWhenTheAppCannotBeToldApart() throws IOException { + assertFailed(AppClasses.derive("", "", null), "names no main class or jar"); + assertFailed(AppClasses.derive("", "Main", null), "no dir or jar on the classpath holds Main.class"); + assertFailed(AppClasses.derive("", "com.acme.app/com.acme.app.Main --x", null), "(java -m)"); + assertFailed(AppClasses.derive("", "./run-app.sh", null), "neither a jar nor a class"); + File noMain = jar(tmp.newFile("lib.jar"), null, "com/acme/shop/A.class"); + assertFailed(AppClasses.derive(noMain.getPath(), noMain.getPath(), null), "names no Main-Class"); + File bootNoStart = jar(tmp.newFile("boot.jar"), "org.springframework.boot.loader.launch.JarLauncher", + "BOOT-INF/classes/com/acme/shop/A.class"); + assertFailed(AppClasses.derive(bootNoStart.getPath(), bootNoStart.getPath(), null), "names no Start-Class"); + assertFailed(AppClasses.derive(noMain.getPath(), "org.springframework.boot.loader.launch.JarLauncher", null), + "no Spring Boot jar on the classpath"); + File notAJar = tmp.newFile("broken.jar"); + Files.write(notAJar.toPath(), "not a zip".getBytes(StandardCharsets.UTF_8)); + assertFailed(AppClasses.derive(notAJar.getPath(), notAJar.getPath(), null), "could not be read"); + } + + @Test + public void aFileSystemRootIsReadOnlyUnderThePackageRoot() throws IOException { + File fsRoot = File.listRoots()[0]; + assertEquals(new File(fsRoot, "org/acme/"), AppClasses.walkFrom(fsRoot, "org/acme/")); + File dir = tmp.newFolder("classes"); + assertEquals(dir, AppClasses.walkFrom(dir, "org/acme/")); + // With no package root, / is not read at all. + assertNull(AppClasses.walkFrom(fsRoot, null)); + assertEquals(dir, AppClasses.walkFrom(dir, null)); + } + + @Test + public void firstTokenIsTheCommandsFirstWordQuotedOrNot() { + assertEquals("/app/my app.jar", AppClasses.firstToken("\"/app/my app.jar\" --x")); + assertEquals("/app/a.jar", AppClasses.firstToken("'/app/a.jar")); + assertEquals("org.acme.Main", AppClasses.firstToken(" org.acme.Main\t--x")); + assertEquals("", AppClasses.firstToken(null)); + assertEquals("", AppClasses.firstToken(" ")); + } + + @Test + public void aLauncherIsNeverTakenForTheApp() throws IOException { + // A jar whose Main-Class is only in a jar its Class-Path names is a + // launcher stub (Quarkus's quarkus-run.jar, Maven's test fork booter). + File dir = tmp.newFolder("stub"); + jar(new File(dir, "lib/boot/runner.jar"), null, "com/acme/runner/Entry.class", "com/acme/runner/Loader.class"); + File stub = new File(dir, "run.jar"); + Manifest manifest = manifest("com.acme.runner.Entry"); + manifest.getMainAttributes().put(Attributes.Name.CLASS_PATH, "lib/boot/runner.jar"); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(stub), manifest)) { + entry(out, "META-INF/app.dat", null); + } + assertFailed(AppClasses.derive(stub.getPath(), stub.getPath(), null), + "its main class com.acme.runner.Entry is not in the jar, which only launches the app"); + + // A known launcher's main class, in its own jar or on -cp. + File start = jar(tmp.newFile("start.jar"), "org.eclipse.jetty.start.Main", "org/eclipse/jetty/start/Main.class"); + assertFailed(AppClasses.derive(start.getPath(), start.getPath(), null), + "is the launcher of org.eclipse.jetty.start, not the app"); + File bootstrap = jar(tmp.newFile("bootstrap.jar"), null, "org/apache/catalina/startup/Bootstrap.class"); + assertFailed(AppClasses.derive(bootstrap.getPath(), "org.apache.catalina.startup.Bootstrap start", null), + "the launcher of org.apache.catalina.startup, not the app"); + assertNull(AppClasses.launcher("org.acme.shop.Main")); + assertEquals("worker.org.gradle", + AppClasses.launcher("worker.org.gradle.process.internal.worker.GradleWorkerMain")); + } + + @Test + public void aThinJarUsesItsStartClass() throws IOException { + File thin = tmp.newFile("thin.jar"); + Manifest manifest = manifest("org.springframework.boot.loader.wrapper.ThinJarWrapper"); + manifest.getMainAttributes().putValue("Start-Class", "com.acme.shop.ShopApplication"); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(thin), manifest)) { + entry(out, "org/springframework/boot/loader/wrapper/ThinJarWrapper.class", null); + entry(out, "com/acme/shop/ShopApplication.class", null); + entry(out, "com/acme/generated/Api.class", null); + } + + AppClasses app = AppClasses.derive(thin.getPath(), thin.getPath(), null); + + assertEquals("com.acme.shop", app.root); + // The wrapper is Spring Boot's, never the app's. + TreeSet expected = new TreeSet<>(Arrays.asList("com/acme/shop/ShopApplication.class", + "com/acme/generated/Api.class")); + assertEquals(expected, scan(app).keySet()); + // The same jar run by Spring Boot's PropertiesLauncher on -cp. + assertEquals(expected, scan(AppClasses.derive(thin.getPath(), + "org.springframework.boot.loader.launch.PropertiesLauncher", null)).keySet()); + + File noStart = jar(tmp.newFile("wrapper.jar"), "org.springframework.boot.loader.wrapper.ThinJarWrapper", + "org/springframework/boot/loader/wrapper/ThinJarWrapper.class"); + assertFailed(AppClasses.derive(noStart.getPath(), noStart.getPath(), null), "names no Start-Class"); + } + + @Test + public void theJarThatHoldsTheMainClassIsTakenWhole() throws IOException { + // Spring Boot's tools extract: app.jar holds the app's classes, and + // its Class-Path names the libs, as URLs. + File dir = tmp.newFolder("extract"); + File model = jar(new File(dir, "lib/demo model.jar"), null, "com/example/demo/model/Item.class", + "com/example/other/Gen.class"); + File jackson = jar(new File(dir, "lib/jackson.jar"), null, "com/fasterxml/jackson/databind/ObjectMapper.class", + "META-INF/maven/com.fasterxml.jackson.core/jackson-databind/pom.properties"); + File app = new File(dir, "app.jar"); + Manifest manifest = manifest("com.example.demo.DemoApplication"); + manifest.getMainAttributes().put(Attributes.Name.CLASS_PATH, "lib/demo%20model.jar lib/jackson.jar"); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(app), manifest)) { + entry(out, "com/example/demo/DemoApplication.class", null); + // Generated code outside the main class's package is the app's too. + entry(out, "org/openapitools/model/Pet.class", null); + entry(out, "META-INF/maven/com.example/demo/pom.properties", null); + } + TreeSet expected = new TreeSet<>(Arrays.asList("com/example/demo/DemoApplication.class", + "org/openapitools/model/Pet.class", "com/example/demo/model/Item.class")); + + AppClasses fromJar = AppClasses.derive(app.getPath(), app.getPath(), null); + + assertEquals("com.example.demo", fromJar.root); + assertEquals(expected, scan(fromJar).keySet()); + // The same jars on a -cp (Jib's packaged mode). + assertEquals(expected, scan(AppClasses.derive(join(app, model, jackson), + "com.example.demo.DemoApplication", null)).keySet()); + + // A jar with a space in its path, run with java -jar unquoted. + File spaced = new File(tmp.newFolder("my app"), "app.jar"); + Files.copy(app.toPath(), spaced.toPath()); + AppClasses unquoted = AppClasses.derive(spaced.getPath(), spaced.getPath() + " --x", null); + assertEquals("java -jar " + spaced.getPath(), unquoted.launch); + assertEquals(new TreeSet<>(Arrays.asList("com/example/demo/DemoApplication.class", + "org/openapitools/model/Pet.class")), scan(unquoted).keySet()); + } + + @Test + public void aShadedJarOnTheClasspathGivesOnlyTheRootsClasses() throws IOException { + File shaded = jar(tmp.newFile("hello.jar"), null, + "com/example/hello/Main.class", "com/example/hello/Res.class", + "org/eclipse/jetty/server/Server.class", + "META-INF/maven/com.example/hello/pom.properties", + "META-INF/maven/org.eclipse.jetty/jetty-server/pom.properties"); + + AppClasses app = AppClasses.derive(shaded.getPath(), "com.example.hello.Main", null); + + assertEquals(new TreeSet<>(Arrays.asList("com/example/hello/Main.class", "com/example/hello/Res.class")), + scan(app).keySet()); + + // With no package root, a shaded jar has nothing to tell apart by. + File flat = jar(tmp.newFile("flat.jar"), "Main", "Main.class", "org/eclipse/jetty/server/Server.class", + "META-INF/maven/acme/flat/pom.properties", + "META-INF/maven/org.eclipse.jetty/jetty-server/pom.properties"); + assertFailed(AppClasses.derive(flat.getPath(), flat.getPath(), null), "a shaded jar whose main class Main"); + assertFailed(AppClasses.derive(flat.getPath(), "Main", null), "from a shaded jar"); + } + + @Test + public void isShadedCountsOtherProjectsPoms() { + assertFalse(AppClasses.isShaded(1, Collections.singleton("org.eclipse.jetty"), "com.acme.shop.Main")); + assertFalse(AppClasses.isShaded(2, Arrays.asList("com.acme", "com.acme.shop.billing"), "com.acme.shop.Main")); + assertFalse(AppClasses.isShaded(2, Collections.singleton("com.acme.shop"), "com.acme.shop.web.Main")); + assertTrue(AppClasses.isShaded(2, Arrays.asList("com.acme.shop", "org.eclipse.jetty"), "com.acme.shop.Main")); + assertTrue(AppClasses.isShaded(2, Collections.singleton("com.acme.shop"), "Main")); + // A sibling under the same first two parts is another project's. + assertTrue(AppClasses.isShaded(2, Collections.singleton("com.acme.tools"), "com.acme.shop.Main")); + } + + @Test + public void withNoPackageRootTheAppsOwnPlacesStillCount() throws IOException { + File classes = tmp.newFolder("flat"); + write(new File(classes, "Main.class")); + write(new File(classes, "util/Helper.class")); + File lib = jar(tmp.newFile("lib.jar"), null, "util/Other.class"); + + AppClasses fromDir = AppClasses.derive(join(classes, lib), "Main", null); + + assertNull(fromDir.root); + assertFalse(fromDir.launch, fromDir.failed); + assertEquals(new TreeSet<>(Arrays.asList("Main.class", "util/Helper.class")), scan(fromDir).keySet()); + + // A Spring Boot jar: BOOT-INF/classes, never its libs. + File model = jar(tmp.newFile("model.jar"), null, "demo/model/Item.class"); + File boot = tmp.newFile("demo.jar"); + Manifest manifest = manifest("org.springframework.boot.loader.launch.JarLauncher"); + manifest.getMainAttributes().putValue("Start-Class", "demo.DemoApplication"); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(boot), manifest)) { + entry(out, "BOOT-INF/classes/demo/DemoApplication.class", null); + entry(out, "BOOT-INF/lib/model.jar", Files.readAllBytes(model.toPath())); + } + AppClasses fromBoot = AppClasses.derive(boot.getPath(), boot.getPath(), null); + assertNull(fromBoot.root); + assertEquals(Collections.singleton("demo/DemoApplication.class"), scan(fromBoot).keySet()); + + // A plain jar: all of it, but none of the jars its Class-Path names. + File plain = tmp.newFile("plain.jar"); + Manifest plainManifest = manifest("Main"); + plainManifest.getMainAttributes().put(Attributes.Name.CLASS_PATH, "lib.jar"); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(plain), plainManifest)) { + entry(out, "Main.class", null); + entry(out, "util/Helper.class", null); + } + assertEquals(new TreeSet<>(Arrays.asList("Main.class", "util/Helper.class")), + scan(AppClasses.derive(plain.getPath(), plain.getPath(), null)).keySet()); + } + + @Test + public void nestedJarsInflateOnlyTheRootsClasses() throws IOException { + // The first entry's data is damaged: a JarInputStream stops there, + // as it must inflate it to step over it; the directory never reads it. + byte[] nested = zip(new String[] {"org/broken/Big.class", "com/acme/shop/A.class", "com/acme/shop/B.class", + "com/acme/shop/CTest.class"}, new boolean[] {false, false, true, false}); + nested[30 + u16(nested, 26) + u16(nested, 28)] = (byte) 0xff; + Map out = new TreeMap<>(); + + assertTrue(AppClasses.scanZipBytes(nested, "com/acme/shop/", "lib.jar!", out, + key -> key.endsWith("Test.class"))); + + assertEquals(new TreeSet<>(Arrays.asList("com/acme/shop/A.class", "com/acme/shop/B.class")), out.keySet()); + assertEquals("com/acme/shop/A.class", new String(out.get("com/acme/shop/A.class").bytes, StandardCharsets.UTF_8)); + assertEquals("com/acme/shop/B.class", new String(out.get("com/acme/shop/B.class").bytes, StandardCharsets.UTF_8)); + assertEquals("lib.jar!com/acme/shop/B.class", out.get("com/acme/shop/B.class").location); + assertFalse(AppClasses.scanZipBytes("not a zip".getBytes(StandardCharsets.UTF_8), "com/", "x!", out, + key -> false)); + } + + @Test + public void aNestedJarItsDirectoryCannotReadIsStreamed() throws IOException { + // Data past the end record, beyond where the directory read looks + // for it: a JarInputStream still reads the entries in order. + byte[] model = zip(new String[] {"com/example/demo/model/Item.class"}, new boolean[] {false}); + byte[] padded = Arrays.copyOf(model, model.length + 70000); + assertFalse(AppClasses.scanZipBytes(padded, "com/", "x!", new TreeMap(), + key -> false)); + File boot = tmp.newFile("demo.jar"); + Manifest manifest = manifest("org.springframework.boot.loader.launch.JarLauncher"); + manifest.getMainAttributes().putValue("Start-Class", "com.example.demo.DemoApplication"); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(boot), manifest)) { + entry(out, "BOOT-INF/classes/com/example/demo/DemoApplication.class", null); + entry(out, "BOOT-INF/lib/model.jar", padded); + } + + Map found = scan(AppClasses.derive(boot.getPath(), boot.getPath(), null)); + + assertEquals(new TreeSet<>(Arrays.asList("com/example/demo/DemoApplication.class", + "com/example/demo/model/Item.class")), found.keySet()); + assertTrue(found.get("com/example/demo/model/Item.class").location, + found.get("com/example/demo/model/Item.class").location + .endsWith("!BOOT-INF/lib/model.jar!com/example/demo/model/Item.class")); + } + + @Test + public void anUnreadableDirIsPassedOver() throws IOException { + File classes = tmp.newFolder("classes"); + write(new File(classes, "com/acme/shop/Main.class")); + File locked = new File(classes, "com/acme/shop/locked"); + write(new File(locked, "Hidden.class")); + Files.setPosixFilePermissions(locked.toPath(), PosixFilePermissions.fromString("---------")); + try { + Assume.assumeFalse("root reads any dir", locked.canRead()); + assertEquals(Collections.singleton("com/acme/shop/Main.class"), + scan(AppClasses.derive(classes.getPath(), "com.acme.shop.Main", null)).keySet()); + } finally { + Files.setPosixFilePermissions(locked.toPath(), PosixFilePermissions.fromString("rwx------")); + } + } + + @Test + public void frameworkLaunchersAreNeverTakenForTheApp() throws IOException { + // A Vert.x fat jar: Main-Class is Vert.x's launcher, Main-Verticle + // the app's. Only the verticle's root counts, never io/vertx. + File vertx = new File(tmp.newFolder("vertx"), "orders-fat.jar"); + Manifest manifest = manifest("io.vertx.core.Launcher"); + manifest.getMainAttributes().putValue("Main-Verticle", "com.acme.orders.MainVerticle"); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(vertx), manifest)) { + for (String name : new String[] {"io/vertx/core/Launcher.class", "io/vertx/core/impl/VertxImpl.class", + "io/netty/channel/Channel.class", "com/acme/orders/MainVerticle.class", + "com/acme/orders/api/OrderHandler.class", "META-INF/maven/io.vertx/vertx-core/pom.properties", + "META-INF/maven/io.netty/netty-transport/pom.properties", + "META-INF/maven/com.acme/orders/pom.properties"}) { + entry(out, name, null); + } + } + AppClasses fat = AppClasses.derive(vertx.getPath(), vertx.getPath(), null); + assertEquals(fat.launch, "com.acme.orders", fat.root); + assertEquals(new TreeSet<>(Arrays.asList("com/acme/orders/MainVerticle.class", + "com/acme/orders/api/OrderHandler.class")), scan(fat).keySet()); + + // The same on -cp, as Vert.x 4 and Vert.x 5 name the verticle. + File core = jar(tmp.newFile("vertx-core.jar"), null, "io/vertx/core/Launcher.class", + "io/vertx/core/http/HttpServer.class", "io/vertx/launcher/application/VertxApplication.class"); + File app = jar(tmp.newFile("orders.jar"), null, "com/acme/orders/MainVerticle.class", + "com/acme/orders/api/OrderHandler.class"); + for (String command : new String[] {"io.vertx.core.Launcher run com.acme.orders.MainVerticle -conf c.json", + "io.vertx.launcher.application.VertxApplication com.acme.orders.MainVerticle"}) { + AppClasses onCp = AppClasses.derive(join(app, core), command, null); + assertEquals(onCp.launch, "com.acme.orders", onCp.root); + assertEquals(new TreeSet<>(Arrays.asList("com/acme/orders/MainVerticle.class", + "com/acme/orders/api/OrderHandler.class")), scan(onCp).keySet()); + } + assertFailed(AppClasses.derive(join(app, core), "io.vertx.core.Launcher", null), + "the launcher of io.vertx.core.Launcher, not the app"); + File noVerticle = jar(tmp.newFile("no-verticle.jar"), "io.vertx.core.Launcher", + "io/vertx/core/Launcher.class"); + assertFailed(AppClasses.derive(noVerticle.getPath(), noVerticle.getPath(), null), + "names no Main-Verticle class"); + + // Play (sbt-native-packager), Helidon MP in a Jib image, Karaf: the + // launcher's package is not the app's, and the app is not found. + File play = jar(tmp.newFile("play-server.jar"), null, "play/core/server/ProdServerStart.class"); + File shop = jar(tmp.newFile("shop.jar"), null, "controllers/HomeController.class", + "com/acme/shop/Module.class"); + assertFailed(AppClasses.derive(join(shop, play), "play.core.server.ProdServerStart", null), + "the launcher of play.core.server, not the app"); + File classes = tmp.newFolder("app", "classes"); + write(new File(classes, "com/acme/shop/GreetResource.class")); + File helidon = jar(tmp.newFile("helidon-microprofile-cdi.jar"), null, + "io/helidon/microprofile/cdi/Main.class"); + assertFailed(AppClasses.derive(join(classes, helidon), "io.helidon.microprofile.cdi.Main", null), + "the launcher of io.helidon.microprofile.cdi, not the app"); + File karaf = jar(tmp.newFile("karaf.jar"), "org.apache.karaf.main.Main", "org/apache/karaf/main/Main.class"); + assertFailed(AppClasses.derive(karaf.getPath(), karaf.getPath(), null), + "is the launcher of org.apache.karaf.main, not the app"); + assertNull(AppClasses.launcher("io.vertx.core.LauncherTest")); + assertNull(AppClasses.verticle("io.vertx.core.Launcher run")); + assertNull(AppClasses.verticle("io.vertx.core.Launcher run -conf")); + } + + @Test + public void aSymlinkedClassFileIsTaken() throws IOException { + File classes = tmp.newFolder("linked"); + write(new File(classes, "com/acme/shop/Main.class")); + File generated = tmp.newFile("Gen.class"); + Files.write(generated.toPath(), "generated".getBytes(StandardCharsets.UTF_8)); + Files.createSymbolicLink(new File(classes, "com/acme/shop/Gen.class").toPath(), generated.toPath()); + // A dangling link is no class. + Files.createSymbolicLink(new File(classes, "com/acme/shop/Gone.class").toPath(), + new File(tmp.getRoot(), "missing.class").toPath()); + + Map found = scan(AppClasses.derive(classes.getPath(), "com.acme.shop.Main", null)); + + assertEquals(new TreeSet<>(Arrays.asList("com/acme/shop/Gen.class", "com/acme/shop/Main.class")), + found.keySet()); + assertEquals("generated", new String(found.get("com/acme/shop/Gen.class").bytes, StandardCharsets.UTF_8)); + } + + @Test + public void aStoredNestedJarIsReadInPlace() throws IOException { + // Spring Boot stores its libs. The first entry's data is damaged: a + // stream stops there, so finding B shows the lib was read by its + // directory, in place. + byte[] nested = zip(new String[] {"org/broken/Big.class", "com/example/demo/model/A.class", + "com/example/demo/model/B.class"}, new boolean[] {false, false, true}); + nested[30 + u16(nested, 26) + u16(nested, 28)] = (byte) 0xff; + File boot = tmp.newFile("stored.jar"); + Manifest manifest = manifest("org.springframework.boot.loader.launch.JarLauncher"); + manifest.getMainAttributes().putValue("Start-Class", "com.example.demo.DemoApplication"); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(boot), manifest)) { + entry(out, "BOOT-INF/classes/com/example/demo/DemoApplication.class", null); + storedEntry(out, "BOOT-INF/lib/model.jar", nested); + } + + Map found = scan(AppClasses.derive(boot.getPath(), boot.getPath(), null)); + + assertEquals(new TreeSet<>(Arrays.asList("com/example/demo/DemoApplication.class", + "com/example/demo/model/A.class", "com/example/demo/model/B.class")), found.keySet()); + assertEquals("com/example/demo/model/B.class", + new String(found.get("com/example/demo/model/B.class").bytes, StandardCharsets.UTF_8)); + } + + @Test + public void aLargeNestedJarIsNeverCopiedIntoTheHeap() throws Exception { + // A 48 MB lib in a JVM of 32 MB of heap: copied whole into the + // heap, it runs the JVM out of memory. + ByteArrayOutputStream lib = new ByteArrayOutputStream(); + try (ZipOutputStream out = new ZipOutputStream(lib)) { + byte[] blob = new byte[48 << 20]; + new java.util.Random(7).nextBytes(blob); + storedEntry(out, "native/libbig.so", blob); + entry(out, "com/example/demo/model/Item.class", null); + } + File boot = tmp.newFile("big.jar"); + Manifest manifest = manifest("org.springframework.boot.loader.launch.JarLauncher"); + manifest.getMainAttributes().putValue("Start-Class", "com.example.demo.DemoApplication"); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(boot), manifest)) { + entry(out, "BOOT-INF/classes/com/example/demo/DemoApplication.class", null); + storedEntry(out, "BOOT-INF/lib/big.jar", lib.toByteArray()); + } + + String javaBin = new File(System.getProperty("java.home"), "bin/java").getPath(); + Process process = new ProcessBuilder(javaBin, "-Xmx32m", "-XX:+ExitOnOutOfMemoryError", "-cp", + System.getProperty("java.class.path"), ScanMain.class.getName(), boot.getPath()) + .redirectErrorStream(true).start(); + String output = new String(readAll(process.getInputStream()), StandardCharsets.UTF_8); + assertTrue(process.waitFor(60, java.util.concurrent.TimeUnit.SECONDS)); + assertEquals(output, 0, process.exitValue()); + assertEquals("com/example/demo/DemoApplication.class,com/example/demo/model/Item.class", output.trim()); + } + + // ScanMain prints the app classes of a jar run with java -jar, for a row + // that needs its own JVM. + public static final class ScanMain { + public static void main(String[] args) { + System.out.print(String.join(",", + scan(AppClasses.derive(args[0], args[0], null)).keySet())); + } + } + + @Test + public void aDirectoryThatDisagreesWithItselfIsStreamed() throws IOException { + byte[] good = zip(new String[] {"com/acme/A.class", "com/acme/B.class", "com/acme/C.class"}, + new boolean[] {false, false, false}); + assertTrue(AppClasses.scanZipBytes(good, "com/", "x!", new TreeMap(), + key -> false)); + int end = good.length - 22; + // The end record's count is 2 of 3, as a jar of 65538 entries + // written without zip64 keeps it. + byte[] fewer = good.clone(); + fewer[end + 10] = 2; + fewer[end + 8] = 2; + assertFalse(AppClasses.scanZipBytes(fewer, "com/", "x!", new TreeMap(), + key -> false)); + // C's size in the directory is 10 of its 16 bytes. + byte[] shorter = good.clone(); + int at = (int) (good.length - 22 - u32(good, end + 12)); + for (int i = 0; i < 2; i++) { + at += 46 + u16(good, at + 28) + u16(good, at + 30) + u16(good, at + 32); + } + shorter[at + 24] = 10; + Map out = new TreeMap<>(); + assertFalse(AppClasses.scanZipBytes(shorter, "com/acme/C", "x!", out, key -> false)); + assertTrue(out.isEmpty()); + // Data before the zip (a fully executable jar's launch script) is + // allowed for. + byte[] script = "#!/bin/bash\nexec java -jar \"$0\"\n".getBytes(StandardCharsets.UTF_8); + byte[] prefixed = new byte[script.length + good.length]; + System.arraycopy(script, 0, prefixed, 0, script.length); + System.arraycopy(good, 0, prefixed, script.length, good.length); + out.clear(); + assertTrue(AppClasses.scanZipBytes(prefixed, "com/", "x!", out, key -> false)); + assertEquals("com/acme/C.class", new String(out.get("com/acme/C.class").bytes, StandardCharsets.UTF_8)); + } + + @Test + public void onlyTheFirstJarThatHoldsTheMainClassIsTakenWhole() throws IOException { + File app = jar(tmp.newFile("shop-2.jar"), null, "com/acme/shop/Main.class", "com/acme/gen/Api.class"); + File old = jar(tmp.newFile("shop-1.jar"), null, "com/acme/shop/Main.class", "com/acme/shop/Old.class", + "org/lib/Util.class"); + + assertEquals(new TreeSet<>(Arrays.asList("com/acme/gen/Api.class", "com/acme/shop/Main.class", + "com/acme/shop/Old.class")), scan(AppClasses.derive(join(app, old), "com.acme.shop.Main", null)) + .keySet()); + } + + @Test + public void aStartClassOutsideBootIsTheMainClass() throws IOException { + File wrapped = tmp.newFile("wrapped.jar"); + Manifest manifest = manifest("com.acme.wrapper.Run"); + manifest.getMainAttributes().putValue("Start-Class", "com.acme.shop.App"); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(wrapped), manifest)) { + entry(out, "com/acme/wrapper/Run.class", null); + entry(out, "com/acme/shop/App.class", null); + } + + assertEquals("com.acme.shop", AppClasses.derive(wrapped.getPath(), wrapped.getPath(), null).root); + } + + // zip writes a zip of the named entries, each holding its own name, + // stored or deflated. + private static byte[] zip(String[] names, boolean[] stored) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ZipOutputStream out = new ZipOutputStream(bytes)) { + for (int i = 0; i < names.length; i++) { + byte[] data = names[i].getBytes(StandardCharsets.UTF_8); + ZipEntry entry = new ZipEntry(names[i]); + if (stored[i]) { + CRC32 crc = new CRC32(); + crc.update(data); + entry.setMethod(ZipEntry.STORED); + entry.setSize(data.length); + entry.setCompressedSize(data.length); + entry.setCrc(crc.getValue()); + } + out.putNextEntry(entry); + out.write(data); + out.closeEntry(); + } + } + return bytes.toByteArray(); + } + + private static int u16(byte[] bytes, int at) { + return (bytes[at] & 0xff) | (bytes[at + 1] & 0xff) << 8; + } + + private static long u32(byte[] bytes, int at) { + return u16(bytes, at) | (long) u16(bytes, at + 2) << 16; + } + + private static byte[] readAll(java.io.InputStream input) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = input.read(buffer)) != -1) { + bytes.write(buffer, 0, read); + } + return bytes.toByteArray(); + } + + // storedEntry writes an entry uncompressed, as Spring Boot writes its + // libs. + private static void storedEntry(ZipOutputStream out, String name, byte[] bytes) throws IOException { + ZipEntry entry = new ZipEntry(name); + CRC32 crc = new CRC32(); + crc.update(bytes); + entry.setMethod(ZipEntry.STORED); + entry.setSize(bytes.length); + entry.setCompressedSize(bytes.length); + entry.setCrc(crc.getValue()); + out.putNextEntry(entry); + out.write(bytes); + out.closeEntry(); + } + + private File springBootJar(String inf) throws IOException { + File model = jar(tmp.newFile("demo-model.jar"), null, "com/example/demo/model/Item.class", + "com/example/demo/DemoApplication.class"); + File jackson = jar(tmp.newFile("jackson.jar"), null, "com/fasterxml/jackson/databind/ObjectMapper.class"); + File app = new File(tmp.newFolder(), "demo." + (inf.startsWith("WEB") ? "war" : "jar")); + Manifest manifest = manifest("org.springframework.boot.loader.launch.JarLauncher"); + manifest.getMainAttributes().putValue("Start-Class", "com.example.demo.DemoApplication"); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(app), manifest)) { + entry(out, "org/springframework/boot/loader/launch/JarLauncher.class", null); + entry(out, inf + "classes/com/example/demo/DemoApplication.class", null); + entry(out, inf + "classes/com/example/other/Helper.class", null); + entry(out, inf + "lib/demo-model.jar", Files.readAllBytes(model.toPath())); + entry(out, inf + "lib/jackson.jar", Files.readAllBytes(jackson.toPath())); + entry(out, "META-INF/maven/com.example/demo/pom.properties", null); + } + return app; + } + + private static void assertFailed(AppClasses app, String reason) { + assertTrue(app.launch, app.failed); + assertNull(app.launch, app.root); + assertTrue(app.launch, app.launch.contains(reason)); + Map found = new LinkedHashMap<>(); + app.scan(found, key -> false); + assertTrue(found.isEmpty()); + } + + private static Map scan(AppClasses app) { + Map found = new LinkedHashMap<>(); + app.scan(found, key -> false); + Map sorted = new TreeMap<>(found); + return sorted; + } + + private static void write(File file) throws IOException { + file.getParentFile().mkdirs(); + Files.write(file.toPath(), file.getName().getBytes(StandardCharsets.UTF_8)); + } + + private static Manifest manifest(String mainClass) { + Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + if (mainClass != null) { + manifest.getMainAttributes().put(Attributes.Name.MAIN_CLASS, mainClass); + } + return manifest; + } + + // jar writes a jar of the named entries, each holding its own name. + private static File jar(File file, String mainClass, String... names) throws IOException { + file.getParentFile().mkdirs(); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(file), manifest(mainClass))) { + for (String name : names) { + entry(out, name, null); + } + } + return file; + } + + private static void entry(ZipOutputStream out, String name, byte[] bytes) throws IOException { + out.putNextEntry(new ZipEntry(name)); + out.write(bytes != null ? bytes : name.getBytes(StandardCharsets.UTF_8)); + out.closeEntry(); + } + + private static String join(File... files) { + StringBuilder classpath = new StringBuilder(); + for (File file : files) { + if (classpath.length() > 0) { + classpath.append(File.pathSeparatorChar); + } + classpath.append(file.getPath()); + } + return classpath.toString(); + } +} diff --git a/keploy-sdk/src/test/java/io/keploy/dedup/AppIndexTest.java b/keploy-sdk/src/test/java/io/keploy/dedup/AppIndexTest.java new file mode 100644 index 00000000..5e48ca4e --- /dev/null +++ b/keploy-sdk/src/test/java/io/keploy/dedup/AppIndexTest.java @@ -0,0 +1,356 @@ +package io.keploy.dedup; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.jar.JarOutputStream; +import java.util.logging.Handler; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.zip.ZipEntry; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.newsclub.net.unix.AFUNIXSocket; +import org.newsclub.net.unix.AFUNIXSocketAddress; + +/** + * The index the agent fingerprints: the app's own classes when they are + * named or derived from the launch, warmed up at the first START; else the + * whole classpath, said at WARNING and never warmed up, as a library's static + * initializer can do anything and a large app's took 54 s. + */ +public class AppIndexTest { + + private static final String[] PROPERTIES = {"java.class.path", "sun.java.command", "user.dir", + "keploy.java.class.dirs", "keploy.java.dedup.warmup.disabled", "keploy.jacoco.port", + "keploy.java.classpath.fallback"}; + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + private final Map saved = new HashMap<>(); + private String savedControlPath; + private String savedDataPath; + private FakeJacoco jacoco; + private final List logged = new CopyOnWriteArrayList<>(); + private final Handler handler = new Handler() { + @Override + public void publish(LogRecord record) { + logged.add(record.getLevel() + " " + record.getMessage()); + } + + @Override + public void flush() { + } + + @Override + public void close() { + } + }; + + @Before + public void setUp() throws IOException { + for (String name : PROPERTIES) { + saved.put(name, System.getProperty(name)); + } + System.clearProperty("keploy.java.class.dirs"); + System.clearProperty("keploy.java.dedup.warmup.disabled"); + System.clearProperty("keploy.java.classpath.fallback"); + // No target/classes or build/classes under the working dir. + System.setProperty("user.dir", tmp.newFolder("cwd").getPath()); + savedControlPath = KeployDedupAgent.controlSocketPath; + savedDataPath = KeployDedupAgent.dataSocketPath; + File sockets = tmp.newFolder("sockets"); + KeployDedupAgent.controlSocketPath = new File(sockets, "control.sock").getAbsolutePath(); + KeployDedupAgent.dataSocketPath = new File(sockets, "data.sock").getAbsolutePath(); + Logger.getLogger(KeployDedupAgent.class.getName()).addHandler(handler); + jacoco = new FakeJacoco(); + System.setProperty("keploy.jacoco.port", String.valueOf(jacoco.server.getLocalPort())); + } + + @After + public void tearDown() throws Exception { + KeployDedupAgent.stop(); + Logger.getLogger(KeployDedupAgent.class.getName()).removeHandler(handler); + KeployDedupAgent.controlSocketPath = savedControlPath; + KeployDedupAgent.dataSocketPath = savedDataPath; + for (String name : PROPERTIES) { + if (saved.get(name) == null) { + System.clearProperty(name); + } else { + System.setProperty(name, saved.get(name)); + } + } + jacoco.close(); + } + + @Test + public void derivedAppClassesAreFingerprintedAndWarmedUp() throws Exception { + File classes = tmp.newFolder("app", "classes"); + write(new File(classes, "com/acme/shop/Main.class")); + write(new File(classes, "com/acme/shop/web/Controller.class")); + System.setProperty("java.class.path", classes.getPath()); + System.setProperty("sun.java.command", "com.acme.shop.Main --port 8080"); + + startAndSendStart(); + + assertLogged("INFO Keploy dedup: fingerprinting the app's 2 classes, from the package com.acme.shop" + + " (java -cp ... com.acme.shop.Main)"); + // The fake classes cannot be loaded: the warmup tried them both. + assertLogged("INFO Keploy dedup: warmed up application classes (initialized=0, skipped=2)"); + assertNotLogged("no warmup"); + } + + @Test + public void configuredClassDirsAreFingerprintedAndWarmedUp() throws Exception { + File classes = tmp.newFolder("module", "classes"); + write(new File(classes, "Main.class")); + System.setProperty("keploy.java.class.dirs", classes.getPath()); + System.setProperty("java.class.path", classes.getPath()); + System.setProperty("sun.java.command", "Main"); + + startAndSendStart(); + + assertLogged("INFO Keploy dedup: fingerprinting the app's 1 classes, from the class dirs [" + classes.getPath() + "]."); + assertLogged("INFO Keploy dedup: warmed up application classes (initialized=0, skipped=1)"); + } + + @Test + public void aJarOnlyAppWithNoSourceTreeIsDerivedByClassNames() throws Exception { + // java -jar app.jar from an empty dir: no source, no build output, + // no git. The index comes from the jar's class names alone. + File withMain = tmp.newFile("shop.jar"); + java.util.jar.Manifest manifest = new java.util.jar.Manifest(); + manifest.getMainAttributes().put(java.util.jar.Attributes.Name.MANIFEST_VERSION, "1.0"); + manifest.getMainAttributes().put(java.util.jar.Attributes.Name.MAIN_CLASS, "com.acme.shop.Main"); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(withMain), manifest)) { + for (String name : new String[] {"com/acme/shop/Main.class", "com/acme/shop/web/Controller.class", + "META-INF/maven/com.acme/shop/pom.properties"}) { + out.putNextEntry(new ZipEntry(name)); + out.write(1); + out.closeEntry(); + } + } + File cwd = new File(System.getProperty("user.dir")); + assertTrue(cwd.list().length == 0); + System.setProperty("java.class.path", withMain.getPath()); + System.setProperty("sun.java.command", withMain.getPath() + " --port 8080"); + + startAndSendStart(); + + assertLogged("INFO Keploy dedup: fingerprinting the app's 2 classes, from the package com.acme.shop" + + " (java -jar " + withMain.getPath() + ")"); + assertLogged("INFO Keploy dedup: warmed up application classes (initialized=0, skipped=2)"); + } + + @Test + public void aSpringBootJarWithNoSourceTreeIsDerivedByClassNames() throws Exception { + // The v1 case: java -jar of a Spring Boot fat jar from an empty dir, + // with no source, build output or git. Its libs are stored, as + // Spring Boot writes them. + File model = jar(tmp.newFile("shop-model.jar"), "com/acme/shop/model/Item.class"); + File jackson = jar(tmp.newFile("jackson.jar"), "com/fasterxml/jackson/databind/ObjectMapper.class"); + File boot = tmp.newFile("shop-boot.jar"); + java.util.jar.Manifest manifest = new java.util.jar.Manifest(); + manifest.getMainAttributes().put(java.util.jar.Attributes.Name.MANIFEST_VERSION, "1.0"); + manifest.getMainAttributes().put(java.util.jar.Attributes.Name.MAIN_CLASS, + "org.springframework.boot.loader.launch.JarLauncher"); + manifest.getMainAttributes().putValue("Start-Class", "com.acme.shop.ShopApplication"); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(boot), manifest)) { + for (String name : new String[] {"org/springframework/boot/loader/launch/JarLauncher.class", + "BOOT-INF/classes/com/acme/shop/ShopApplication.class", + "BOOT-INF/classes/com/acme/shop/web/Controller.class"}) { + out.putNextEntry(new ZipEntry(name)); + out.write(1); + out.closeEntry(); + } + for (File lib : new File[] {model, jackson}) { + byte[] bytes = java.nio.file.Files.readAllBytes(lib.toPath()); + ZipEntry entry = new ZipEntry("BOOT-INF/lib/" + lib.getName()); + java.util.zip.CRC32 crc = new java.util.zip.CRC32(); + crc.update(bytes); + entry.setMethod(ZipEntry.STORED); + entry.setSize(bytes.length); + entry.setCompressedSize(bytes.length); + entry.setCrc(crc.getValue()); + out.putNextEntry(entry); + out.write(bytes); + out.closeEntry(); + } + } + File cwd = new File(System.getProperty("user.dir")); + assertTrue(cwd.list().length == 0); + System.setProperty("java.class.path", boot.getPath()); + System.setProperty("sun.java.command", boot.getPath()); + + startAndSendStart(); + + assertLogged("INFO Keploy dedup: fingerprinting the app's 3 classes, from the package com.acme.shop" + + " (java -jar " + boot.getPath() + ")"); + } + + @Test + public void derivedAppClassesNeedNoClasspathFallback() throws Exception { + File classes = tmp.newFolder("app", "classes"); + write(new File(classes, "com/acme/shop/Main.class")); + System.setProperty("java.class.path", classes.getPath()); + System.setProperty("sun.java.command", "com.acme.shop.Main"); + System.setProperty("keploy.java.classpath.fallback", "false"); + + startAndSendStart(); + + assertLogged("INFO Keploy dedup: fingerprinting the app's 1 classes, from the package com.acme.shop" + + " (java -cp ... com.acme.shop.Main)"); + assertLogged("INFO Keploy dedup: warmed up application classes (initialized=0, skipped=1)"); + } + + @Test + public void aMainClassWithNoPackageStillHasItsDir() throws Exception { + File classes = tmp.newFolder("flat"); + write(new File(classes, "Main.class")); + write(new File(classes, "lib/Helper.class")); + System.setProperty("java.class.path", classes.getPath()); + System.setProperty("sun.java.command", "Main"); + + startAndSendStart(); + + assertLogged("INFO Keploy dedup: fingerprinting the app's 2 classes, from java -cp ... Main."); + assertLogged("INFO Keploy dedup: warmed up application classes (initialized=0, skipped=2)"); + } + + @Test + public void wholeClasspathIsSaidAndNeverWarmedUp() throws Exception { + // Tomcat's launcher: its package is not the app's. + File classes = tmp.newFolder("webapps"); + write(new File(classes, "com/acme/shop/Servlet.class")); + File bootstrap = jar(tmp.newFile("bootstrap.jar"), "org/apache/catalina/startup/Bootstrap.class"); + System.setProperty("java.class.path", classes.getPath() + File.pathSeparator + bootstrap.getPath()); + System.setProperty("sun.java.command", "org.apache.catalina.startup.Bootstrap start"); + + startAndSendStart(); + + assertLogged("WARNING Keploy dedup could not tell the app's own classes from its libraries (java -cp ..." + + " org.apache.catalina.startup.Bootstrap, the launcher of org.apache.catalina.startup, not the" + + " app), so it fingerprints all 2 classes of the whole classpath"); + assertLogged("INFO Keploy dedup: no warmup, as the index is not the app's own classes alone" + + " (from the whole classpath)"); + assertNotLogged("warmed up"); + } + + @Test + public void nothingFoundIsSaidAtWarning() throws Exception { + File classes = tmp.newFolder("webapps"); + write(new File(classes, "com/acme/shop/Servlet.class")); + File bootstrap = jar(tmp.newFile("bootstrap.jar"), "org/apache/catalina/startup/Bootstrap.class"); + System.setProperty("java.class.path", classes.getPath() + File.pathSeparator + bootstrap.getPath()); + System.setProperty("sun.java.command", "org.apache.catalina.startup.Bootstrap start"); + System.setProperty("keploy.java.classpath.fallback", "false"); + + startAndSendStart(); + + assertLogged("WARNING Keploy dedup found none of the app's classes (java -cp ..." + + " org.apache.catalina.startup.Bootstrap, the launcher of org.apache.catalina.startup, not the" + + " app), nor any class in the jar, and KEPLOY_JAVA_CLASSPATH_FALLBACK=false keeps it off the" + + " classpath, so no test's coverage is measured. Set KEPLOY_JAVA_CLASS_DIRS"); + assertNotLogged("fingerprint"); + assertNotLogged("warmed up"); + } + + @Test + public void aMainClassOffTheClasspathFallsBackAndSaysWhere() throws Exception { + // The launch names com.acme.shop.Main, but the one jar does not hold it. + File jar = jar(tmp.newFile("app.jar"), "org/other/Main.class"); + System.setProperty("java.class.path", jar.getPath()); + System.setProperty("sun.java.command", "com.acme.shop.Main"); + + startAndSendStart(); + + assertLogged("WARNING Keploy dedup could not tell the app's own classes from its libraries (java -cp ..." + + " com.acme.shop.Main, but no dir or jar on the classpath holds com/acme/shop/Main.class), so it" + + " fingerprints all 1 classes of the whole jar [" + jar.getPath() + "]"); + assertLogged("INFO Keploy dedup: no warmup"); + } + + @Test + public void aRootWithNoClassFallsBackAndSaysWhere() throws Exception { + // The main class's dir holds only what the index never takes. + File classes = tmp.newFolder("tests"); + write(new File(classes, "com/acme/shop/MainTest.class")); + File other = jar(tmp.newFile("other.jar"), "org/other/Lib.class"); + System.setProperty("java.class.path", classes.getPath() + File.pathSeparator + other.getPath()); + System.setProperty("sun.java.command", "com.acme.shop.MainTest"); + + startAndSendStart(); + + assertLogged("WARNING Keploy dedup could not tell the app's own classes from its libraries (no class found" + + " from the package com.acme.shop (java -cp ... com.acme.shop.MainTest)), so it fingerprints all 1" + + " classes of the whole classpath"); + } + + private void startAndSendStart() throws IOException { + assertTrue(KeployDedupAgent.start()); + assertEquals("ACK proto=2\n", send("START test-set-0/test-1")); + } + + private void assertLogged(String prefix) { + for (String line : logged) { + if (line.startsWith(prefix)) { + return; + } + } + throw new AssertionError("nothing logged starts with: " + prefix + "\nlogged: " + logged); + } + + private void assertNotLogged(String part) { + for (String line : logged) { + assertTrue("logged: " + line, !line.contains(part)); + } + } + + private static File jar(File file, String... names) throws IOException { + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(file))) { + for (String name : names) { + out.putNextEntry(new ZipEntry(name)); + out.write(1); + out.closeEntry(); + } + } + return file; + } + + private static void write(File file) throws IOException { + file.getParentFile().mkdirs(); + Files.write(file.toPath(), new byte[] {1}); + } + + private static String send(String command) throws IOException { + try (AFUNIXSocket socket = AFUNIXSocket.newInstance()) { + socket.connect(AFUNIXSocketAddress.of(new File(KeployDedupAgent.controlSocketPath)), 3000); + socket.setSoTimeout(30000); + OutputStream out = socket.getOutputStream(); + out.write((command + "\n").getBytes(StandardCharsets.UTF_8)); + out.flush(); + InputStream in = socket.getInputStream(); + StringBuilder answer = new StringBuilder(); + int b; + while ((b = in.read()) != -1) { + answer.append((char) b); + } + return answer.toString(); + } + } +} From 998c61806100b74da045376a6b8b2e4f27a47f83 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Thu, 19 Nov 2026 22:20:37 +0530 Subject: [PATCH 2/5] feat(dedup): start JaCoCo from keploy-sdk.jar on the app's classes only A JaCoCo agent that the command attaches instruments every class the JVM loads. On a large app nearly all of those are library classes, so most of the instrumentation time at startup goes to code that dedup then filters out of every fingerprint. JaCoCo reads includes= only from its own agent options, in its premain, so nothing outside the JVM can set it from the classes the SDK finds from the launch. keploy-sdk.jar now carries jacocoagent.jar (org.jacoco:org.jacoco.agent 0.8.15, unchanged) and, when the JVM has no JaCoCo agent of its own, starts it in premain, before the app's main runs. It does so after the control socket is bound (or, on TCP, the client made) and before the worker that reads commands starts, so a START that comes early waits for JaCoCo instead of resetting one not yet started: - it reads the app's class index there (the first START read it before); - it writes jacocoagent.jar into a new temp dir only this user can read, appends it to the system classpath as the JVM does a -javaagent jar, and runs its premain with output=none and includes= the index's classes: the first three parts of each package ("org.acme.web.*"), none under another, and the exact name of a class in a package of fewer than two parts. Every indexed class matches, and the coverage dump is already cut to the index, so nothing dedup reads is lost; - when the index is not the app's own classes alone (the whole jar or the whole classpath), it passes no includes=, so JaCoCo instruments every class, as an attached one does; - when the index is empty, it starts no JaCoCo and says so at WARNING: no class could pass the index's cut; - if JaCoCo cannot start, it says so at WARNING and the app runs with no coverage measured. With KEPLOY_JAVA_DEDUP_DISABLED=true, or when the socket cannot be bound, the SDK starts no JaCoCo. A JaCoCo agent the command attaches, before or after keploy-sdk.jar, on the command line or in JAVA_TOOL_OPTIONS, is left to run alone, said at INFO. One before it is on the system classpath already; one after it is not until its own premain runs, so it is found by its jar among the JVM's arguments. A second JaCoCo agent would fail the JVM's start on Java 9 and later (it defines java.lang.$JaCoCo again), and on Java 8 log an error for every class it loads. So k8s, where the proxy attaches JaCoCo, runs as it did, with no index read before main. Limits of this change: the index is read before main, so startup grows by that read (on the whole-classpath fallback, the whole classpath); the temp dir is removed when the JVM exits, except on Windows, where the JVM holds the jar open, and a JVM killed with SIGKILL leaves it behind. META-INF/THIRD-PARTY-NOTICES.txt names JaCoCo (EPL-2.0, with where its source is) and ASM (with its BSD 3-Clause text), and the other software the jar carries. The README now runs the app with keploy-sdk.jar alone, and says to remove keploy-sdk from the app's dependencies. jacoco.version sets both JaCoCo artifacts. Rows: EmbeddedJacocoTest (includes= takes in every indexed class and no library, none for a fallback index, no JaCoCo for an empty one, the options, detection on the classpath and among the arguments, and six JVMs of their own: the SDK alone, JaCoCo after it on the command line and in JAVA_TOOL_OPTIONS, JaCoCo before it in JAVA_TOOL_OPTIONS, dedup turned off, and a temp dir JaCoCo cannot start from). The smoke script runs two modes with keploy-sdk.jar alone. Signed-off-by: slayerjain --- README.md | 49 +++- keploy-sdk/pom.xml | 11 +- .../java/io/keploy/dedup/EmbeddedJacoco.java | 175 ++++++++++++ .../io/keploy/dedup/KeployDedupAgent.java | 60 +++- .../META-INF/THIRD-PARTY-NOTICES.txt | 49 ++++ .../io/keploy/dedup/EmbeddedJacocoTest.java | 266 ++++++++++++++++++ .../org/example/embedded/EmbeddedApp.java | 37 +++ scripts/smoke-javaagent.sh | 29 ++ 8 files changed, 658 insertions(+), 18 deletions(-) create mode 100644 keploy-sdk/src/main/java/io/keploy/dedup/EmbeddedJacoco.java create mode 100644 keploy-sdk/src/main/resources/META-INF/THIRD-PARTY-NOTICES.txt create mode 100644 keploy-sdk/src/test/java/io/keploy/dedup/EmbeddedJacocoTest.java create mode 100644 keploy-sdk/src/test/java/org/example/embedded/EmbeddedApp.java diff --git a/README.md b/README.md index 1b9ef46d..5cef3f38 100644 --- a/README.md +++ b/README.md @@ -61,23 +61,41 @@ Download the `keploy-sdk` jar and keep it outside your application dependencies. ``` -The SDK no longer has to be added to `dependencies`, and application code should not import `io.keploy.*` classes for dynamic deduplication. +Do not add `io.keploy:keploy-sdk` to the app's dependencies, and remove it if it is there: it must not be on the +app's classpath, where an older copy would run in place of the attached `keploy-sdk.jar` and start no JaCoCo, so no +test's coverage would be measured. Application code should not import `io.keploy.*` classes for dynamic deduplication. -### 2. Run the App with the Keploy and JaCoCo Java Agents +### 2. Run the App with the Keploy Java Agent -The dedup agent reads coverage in-process via JaCoCo's runtime API (`org.jacoco.agent.rt.RT.getAgent()`), so attaching the JaCoCo Java agent is the only runtime requirement in the common cases below: - -- Maven/Gradle dev runs where application classes are under `target/classes` or `build/classes/java/main` -- packaged `java -jar` runs where the application classes live inside the executable jar +`keploy-sdk.jar` is the only agent the app needs: ```bash -java \ - -javaagent:/path/to/keploy-sdk.jar \ - -javaagent:/path/to/jacocoagent.jar \ - -jar your-app.jar +java -javaagent:/path/to/keploy-sdk.jar -jar your-app.jar ``` -If the in-process API is unavailable (for example because the JaCoCo agent is loaded into an isolated classloader), the SDK transparently falls back to JaCoCo's TCP server mode. To use the fallback explicitly, start JaCoCo in `tcpserver` mode and set `KEPLOY_JACOCO_HOST` / `KEPLOY_JACOCO_PORT`: +It carries the JaCoCo agent (`jacocoagent.jar`, JaCoCo 0.8.15, EPL-2.0; see +`META-INF/THIRD-PARTY-NOTICES.txt` in the jar) and starts it itself, before the +app's `main`, with `includes=` set to the app's own classes, which it finds from +how the JVM was launched (Spring Boot, Jib, shaded and plain jars, class dirs; +`KEPLOY_JAVA_CLASS_DIRS` overrides it). JaCoCo then instruments only those +classes, not every library class the app loads. The startup log says which: + +``` +INFO: Keploy dedup: started JaCoCo 0.8.15 on the app's classes alone (includes=com.example.shop.*) +``` + +When the SDK cannot tell the app's classes from its libraries, it says so at +WARNING and starts JaCoCo on every class; set `KEPLOY_JAVA_CLASS_DIRS` to the +app's class dirs and jars to fix that. + +If the command attaches a JaCoCo agent of its own (`-javaagent:jacocoagent.jar`, +in any order, or in `JAVA_TOOL_OPTIONS`), the SDK starts none and reads coverage +from that one, through JaCoCo's in-process runtime API +(`org.jacoco.agent.rt.RT.getAgent()`). That JaCoCo instruments every class its +own options take in. If the in-process API is unavailable (for example because +the JaCoCo agent is loaded into an isolated classloader), the SDK falls back to +JaCoCo's TCP server mode. To use the fallback explicitly, start JaCoCo in +`tcpserver` mode and set `KEPLOY_JACOCO_HOST` / `KEPLOY_JACOCO_PORT`: ```bash java \ @@ -91,7 +109,7 @@ java \ Run replay with dynamic dedup enabled: ```bash -keploy test -c "java -javaagent:/path/to/keploy-sdk.jar -javaagent:/path/to/jacocoagent.jar -jar your-app.jar" \ +keploy test -c "java -javaagent:/path/to/keploy-sdk.jar -jar your-app.jar" \ --dedup \ --language java ``` @@ -128,7 +146,7 @@ supported ceiling is the lower of two limits — not one: | Component | Role | Reads up to | | --- | --- | --- | -| `jacocoagent.jar` (k8s-proxy init image) | analyzes coverage | Java 26 (JaCoCo 0.8.15) | +| `jacocoagent.jar` (carried in `keploy-sdk.jar`; on k8s, the k8s-proxy init image's) | analyzes coverage | Java 26 (JaCoCo 0.8.15) | | `keploy-sdk.jar` (this repo) | instruments the app, via the ASM shaded in through `org.jacoco:org.jacoco.core` | Java 26 (jacoco.core 0.8.15) | Both currently sit at **Java 26**, so that is the effective ceiling. @@ -153,7 +171,10 @@ reason instead of failing silently. ### Raising it -1. Bump `org.jacoco:org.jacoco.core` in `keploy-sdk/pom.xml`, and cut a release. +1. Bump `jacoco.version` in `keploy-sdk/pom.xml` (both `org.jacoco.core` and the + `jacocoagent.jar` the SDK carries), change the JaCoCo and ASM versions named in + `keploy-sdk/src/main/resources/META-INF/THIRD-PARTY-NOTICES.txt` and in this + README to match, and cut a release. 2. In k8s-proxy, bump `KEPLOY_SDK_VERSION` (`build/dedup-jars/build.sh`) to that release, and set `BundledJaCoCoVersion`, `JaCoCoOfficialMaxJava` and `SDKShadedASMMaxJava` in `pkg/platform/dedupinject/jacocompat.go`. diff --git a/keploy-sdk/pom.xml b/keploy-sdk/pom.xml index d254fcda..1e1d3714 100644 --- a/keploy-sdk/pom.xml +++ b/keploy-sdk/pom.xml @@ -19,6 +19,7 @@ UTF-8 UTF-8 + 0.8.15 @@ -67,7 +68,15 @@ ASM is an independent ceiling on the Java bytecode dynamic dedup can instrument. While this sat on 0.8.12 it, not the jacocoagent in the init image, was what capped dedup at Java 22. --> - 0.8.15 + ${jacoco.version} + + + + org.jacoco + org.jacoco.agent + ${jacoco.version} com.kohlschutter.junixsocket diff --git a/keploy-sdk/src/main/java/io/keploy/dedup/EmbeddedJacoco.java b/keploy-sdk/src/main/java/io/keploy/dedup/EmbeddedJacoco.java new file mode 100644 index 00000000..24d7cee7 --- /dev/null +++ b/keploy-sdk/src/main/java/io/keploy/dedup/EmbeddedJacoco.java @@ -0,0 +1,175 @@ +package io.keploy.dedup; + +import org.jacoco.agent.AgentJar; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.lang.instrument.Instrumentation; +import java.nio.file.Files; +import java.util.Collection; +import java.util.List; +import java.util.TreeSet; +import java.util.jar.Attributes; +import java.util.jar.JarFile; +import java.util.jar.Manifest; + +/** + * EmbeddedJacoco starts the JaCoCo agent that keploy-sdk.jar carries, so + * that {@code -javaagent:keploy-sdk.jar} is all an app needs, and has it + * instrument only the app's own classes. + *

+ * JaCoCo reads includes= only from its own agent options, in its premain: + * a JaCoCo agent the command attaches gets the options the command gives + * it, and with none it instruments every class it loads (about 105k of + * Fineract's 112k, against its 7k own, at about 15 s of startup). So the + * SDK starts JaCoCo itself, with includes= taking in every class in the + * app's index ({@link #includes(boolean, Collection)}), and leaves alone a + * JaCoCo agent the command attaches. + *

+ * JaCoCo is Eclipse Public License 2.0 software + * (https://github.com/jacoco/jacoco), carried unchanged as the + * {@code jacocoagent.jar} of {@code org.jacoco:org.jacoco.agent}; see + * {@code META-INF/THIRD-PARTY-NOTICES.txt} in keploy-sdk.jar. + */ +final class EmbeddedJacoco { + + // A class every JaCoCo agent jar holds. + static final String RUNTIME_CLASS = "org/jacoco/agent/rt/RT.class"; + private static final String JAVAAGENT = "-javaagent:"; + // A pattern takes in at most this many parts of a class's package, so + // that an app over many packages needs only a few patterns; a package of + // fewer than two parts ("Main", "app.Main") is never a pattern, as + // "com.*" would take in every library under com. + private static final int PATTERN_PARTS = 3; + + private EmbeddedJacoco() { + } + + /** + * attached says whether the JVM has a JaCoCo agent of its own: one the + * command or JAVA_TOOL_OPTIONS attaches with -javaagent. An agent before + * keploy-sdk.jar is on the system classpath already; one after it is + * not until its own premain, so it is found by its jar among the JVM's + * arguments (the RuntimeMXBean's, JAVA_TOOL_OPTIONS' among them). + */ + static boolean attached(ClassLoader system, List jvmArguments) { + if (system.getResource(RUNTIME_CLASS) != null) { + return true; + } + for (String argument : jvmArguments) { + if (!argument.startsWith(JAVAAGENT)) { + continue; + } + String path = argument.substring(JAVAAGENT.length()); + int options = path.indexOf('='); + try (JarFile jar = new JarFile(options < 0 ? path : path.substring(0, options))) { + if (jar.getEntry(RUNTIME_CLASS) != null) { + return true; + } + } catch (IOException e) { + // The JVM cannot load an agent from it either. + } + } + return false; + } + + /** + * includes is JaCoCo's includes= for an index of the app's own classes, + * given by VM name ("org/acme/web/Foo"): a pattern of the first three + * parts of each class's package ("org.acme.web.*"), none under another + * ("org.acme.*" takes in "org.acme.web.*"), and the class's own name for + * a class in a package of fewer than two parts. Every class given + * matches it. It is "", so that JaCoCo instruments every class, for an + * index that is not the app's own classes alone (the whole jar or the + * whole classpath, which would need a pattern for nearly every library). + * It is null for an empty index: no class passes its cut, so no JaCoCo + * agent is started. + */ + static String includes(boolean appOnly, Collection classNames) { + if (classNames.isEmpty()) { + return null; + } + if (!appOnly) { + return ""; + } + TreeSet patterns = new TreeSet<>(); + for (String vmName : classNames) { + String name = vmName.replace('/', '.'); + String[] parts = name.split("\\."); + int packageParts = parts.length - 1; + if (packageParts < 2) { + patterns.add(name); + continue; + } + StringBuilder pattern = new StringBuilder(); + for (int i = 0; i < Math.min(packageParts, PATTERN_PARTS); i++) { + pattern.append(parts[i]).append('.'); + } + patterns.add(pattern.append('*').toString()); + } + StringBuilder joined = new StringBuilder(); + for (String pattern : patterns) { + if (underAnother(pattern, patterns)) { + continue; + } + if (joined.length() > 0) { + joined.append(':'); + } + joined.append(pattern); + } + return joined.toString(); + } + + // underAnother says whether a pattern of three parts ("a.b.c.*") is + // taken in by the pattern of its first two ("a.b.*"). + // A class's own name has at most two parts, so it is never taken in. + private static boolean underAnother(String pattern, Collection patterns) { + String[] parts = pattern.split("\\."); + return parts.length == PATTERN_PARTS + 1 + && patterns.contains(parts[0] + "." + parts[1] + ".*"); + } + + /** + * options are the JaCoCo agent options the SDK starts it with: no output + * of its own (the SDK reads the counters in-process, through JaCoCo's + * RT.getAgent()), and includes= when there are any. + */ + static String options(String includes) { + return includes.isEmpty() ? "output=none" : "output=none,includes=" + includes; + } + + /** + * start writes the JaCoCo agent jar into a new temp dir only this user + * can read, puts it on the system classpath as the JVM does a + * -javaagent jar, and runs its premain with options. It returns + * JaCoCo's version. The jar is removed when the JVM exits, except on + * Windows, where the JVM holds it open. + */ + static String start(Instrumentation instrumentation, String options) throws Exception { + File dir = Files.createTempDirectory("keploy-jacoco-").toFile(); + // Removed in the reverse of this order: the jar, then its dir. + dir.deleteOnExit(); + File jar = new File(dir, "jacocoagent.jar"); + jar.deleteOnExit(); + try (InputStream input = AgentJar.getResourceAsStream()) { + Files.copy(input, jar.toPath()); + } + String premain; + String version; + try (JarFile jarFile = new JarFile(jar)) { + Manifest manifest = jarFile.getManifest(); + Attributes attributes = manifest == null ? new Attributes() : manifest.getMainAttributes(); + premain = attributes.getValue("Premain-Class"); + version = attributes.getValue("Implementation-Version"); + if (premain == null) { + throw new IllegalStateException("the JaCoCo agent jar names no Premain-Class"); + } + instrumentation.appendToSystemClassLoaderSearch(jarFile); + } + Class.forName(premain, true, ClassLoader.getSystemClassLoader()) + .getMethod("premain", String.class, Instrumentation.class) + .invoke(null, options, instrumentation); + return version == null ? "" : version; + } +} diff --git a/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java b/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java index c171184a..a61fe5f3 100644 --- a/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java +++ b/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java @@ -21,6 +21,7 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.lang.instrument.Instrumentation; +import java.lang.management.ManagementFactory; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.InetAddress; @@ -136,7 +137,7 @@ private KeployDedupAgent() { * @param instrumentation JVM instrumentation handle */ public static void premain(String agentArgs, Instrumentation instrumentation) { - start(); + start(new CoverageIndex(), instrumentation); } /** @@ -155,6 +156,17 @@ public static void agentmain(String agentArgs, Instrumentation instrumentation) * @return {@code true} when the agent is running or was already started */ public static boolean start() { + return start(new CoverageIndex(), null); + } + + // start starts the agent on index. From premain (instrumentation not + // null) it also starts the JaCoCo agent keploy-sdk.jar carries, after the + // socket is bound and before the worker that reads commands runs: a START + // that comes earlier waits in the socket's backlog, so it never resets a + // JaCoCo not yet started, or warms up app classes before JaCoCo can + // instrument them. A start that finds the agent started already starts + // no JaCoCo: the call that started the agent did. + private static boolean start(CoverageIndex index, Instrumentation instrumentation) { if (isDisabled()) { return false; } @@ -168,8 +180,7 @@ public static boolean start() { } CoverageCollector collector = new CoverageCollector( - new JacocoClient(resolveHost(), resolvePort()), - new CoverageIndex()); + new JacocoClient(resolveHost(), resolvePort()), index); CoverageWorker worker; String threadName; @@ -227,6 +238,9 @@ public static boolean start() { threadName = "keploy-java-dedup-control"; } + if (instrumentation != null) { + startJacoco(instrumentation, index); + } Thread thread = new Thread(worker, threadName); thread.setDaemon(true); coverageThread = thread; @@ -235,6 +249,46 @@ public static boolean start() { return true; } + // startJacoco starts the JaCoCo agent keploy-sdk.jar carries, with + // includes= the app's classes in index (EmbeddedJacoco), unless the JVM + // has a JaCoCo agent of its own or the index holds no class. It runs in + // premain, before the app's main loads any app class, so it reads the + // index there: the app's classes found from the launch, with their + // bytes, which the first START read before. + private static void startJacoco(Instrumentation instrumentation, CoverageIndex index) { + try { + if (EmbeddedJacoco.attached(ClassLoader.getSystemClassLoader(), + ManagementFactory.getRuntimeMXBean().getInputArguments())) { + log(Level.INFO, "Keploy dedup: this JVM has a JaCoCo agent attached, so Keploy reads coverage" + + " from it, and it instruments every class its own options take in. Attach only" + + " keploy-sdk.jar to have JaCoCo instrument the app's own classes alone", null); + return; + } + List names = new ArrayList<>(); + for (ClassEntry entry : index.entries()) { + names.add(entry.className); + } + String includes = EmbeddedJacoco.includes(index.isAppOnly(), names); + if (includes == null) { + // Said at WARNING, as the index read was: no class can pass + // the index's cut, so JaCoCo would only slow the app down. + log(Level.WARNING, "Keploy dedup starts no JaCoCo agent: it found no class to measure", null); + return; + } + String version = EmbeddedJacoco.start(instrumentation, EmbeddedJacoco.options(includes)); + log(Level.INFO, "Keploy dedup: started JaCoCo " + version + (includes.isEmpty() + ? " on every class" + : " on the app's " + names.size() + " classes alone (includes=" + + (includes.length() > 300 ? includes.substring(0, 300) + "..." : includes) + ")"), null); + } catch (Throwable t) { + // Never let premain fail the app's start: run it unmeasured. This + // must stay Throwable: an Error (NoClassDefFoundError for JaCoCo's + // AgentJar, a LinkageError from its premain) would abort the JVM. + log(Level.WARNING, "Keploy dedup could not start the JaCoCo agent keploy-sdk.jar carries, so no" + + " test's coverage is measured: attach jacocoagent.jar with -javaagent to measure it", t); + } + } + /** * Returns whether the background control socket listener is active. * diff --git a/keploy-sdk/src/main/resources/META-INF/THIRD-PARTY-NOTICES.txt b/keploy-sdk/src/main/resources/META-INF/THIRD-PARTY-NOTICES.txt new file mode 100644 index 00000000..442d0e08 --- /dev/null +++ b/keploy-sdk/src/main/resources/META-INF/THIRD-PARTY-NOTICES.txt @@ -0,0 +1,49 @@ +keploy-sdk.jar (Apache License 2.0) carries the following third-party software. + +JaCoCo 0.8.15 - Eclipse Public License 2.0 (https://www.eclipse.org/legal/epl-2.0/) + Copyright (c) 2009, 2026 Mountainminds GmbH & Co. KG and Contributors + JaCoCo's source code is available under the EPL-2.0 at + https://github.com/jacoco/jacoco, tag v0.8.15. + - jacocoagent.jar, unchanged, from org.jacoco:org.jacoco.agent:0.8.15; keploy-sdk + starts it when the JVM attaches no JaCoCo agent of its own. + - the classes of org.jacoco:org.jacoco.core:0.8.15, unchanged. + +ASM 9.10.1 - BSD 3-Clause License + Source: https://gitlab.ow2.org/asm/asm + - the classes of org.ow2.asm:asm, asm-commons and asm-tree 9.10.1, unchanged + (under org/objectweb/asm), which org.jacoco.core depends on, and the copy + inside jacocoagent.jar. ASM is subject to the following license: + +ASM: a very small and fast Java bytecode manipulation framework +Copyright (c) 2000-2011 INRIA, France Telecom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. + +Gson 2.9.1 - Apache License 2.0 + Source: https://github.com/google/gson + +junixsocket 2.10.1 - Apache License 2.0 + Source: https://github.com/kohlschutter/junixsocket diff --git a/keploy-sdk/src/test/java/io/keploy/dedup/EmbeddedJacocoTest.java b/keploy-sdk/src/test/java/io/keploy/dedup/EmbeddedJacocoTest.java new file mode 100644 index 00000000..92cb3d1f --- /dev/null +++ b/keploy-sdk/src/test/java/io/keploy/dedup/EmbeddedJacocoTest.java @@ -0,0 +1,266 @@ +package io.keploy.dedup; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.jar.Attributes; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import java.util.zip.ZipEntry; +import org.example.embedded.EmbeddedApp; +import org.jacoco.agent.AgentJar; +import org.jacoco.core.runtime.WildcardMatcher; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class EmbeddedJacocoTest { + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + @Test + public void includesTakeInEveryIndexedClassAndNoLibrary() { + List app = Arrays.asList( + "org/apache/fineract/portfolio/loan/Loan", + "org/apache/fineract/portfolio/loan/Loan$1", + "org/apache/fineract/infrastructure/core/boot/Main", + "org/acme/A", + "org/acme/web/B", + "org/acme/web/deep/C", + "com/acme/gen/api/Model", + "acme/Bar", + "Main", + "Main$Inner"); + String includes = EmbeddedJacoco.includes(true, app); + + assertEquals("Main:Main$Inner:acme.Bar:com.acme.gen.*:org.acme.*:org.apache.fineract.*", includes); + WildcardMatcher matcher = new WildcardMatcher(includes); + for (String name : app) { + assertTrue(name, matcher.matches(name.replace('/', '.'))); + } + // Libraries, and the app's neighbours in name, stay out. + for (String library : new String[] {"org.springframework.web.Servlet", "org.apache.commons.Lang", + "com.acme.other.X", "com.acmex.gen.Y", "acme.Baz", "acme.sub.Q", "MainX", "org.acmex.Z", + "com.google.gson.Gson"}) { + assertFalse(library, matcher.matches(library)); + } + } + + @Test + public void includesAreNoneUnlessTheIndexIsTheAppsAlone() { + // The whole jar or the whole classpath: JaCoCo instruments every + // class, as it did before the SDK started it. + assertEquals("", EmbeddedJacoco.includes(false, Collections.singletonList("org/acme/web/B"))); + } + + @Test + public void anEmptyIndexStartsNoJacoco() { + // No class passes the index's cut, so JaCoCo would only slow the app. + assertNull(EmbeddedJacoco.includes(true, Collections.emptyList())); + assertNull(EmbeddedJacoco.includes(false, Collections.emptyList())); + } + + @Test + public void optionsCarryIncludesOnlyWhenThereAreAny() { + assertEquals("output=none", EmbeddedJacoco.options("")); + assertEquals("output=none,includes=org.acme.*:Main", EmbeddedJacoco.options("org.acme.*:Main")); + } + + @Test + public void aJacocoAgentOnTheClasspathOrAmongTheArgumentsCountsAsAttached() throws Exception { + File jacoco = tmp.newFile("jacocoagent.jar"); + AgentJar.extractTo(jacoco); + File other = sdkJar(); + List none = Collections.emptyList(); + try (URLClassLoader withJacoco = new URLClassLoader(new URL[] {jacoco.toURI().toURL()}, null); + URLClassLoader without = new URLClassLoader(new URL[] {tmp.getRoot().toURI().toURL()}, null)) { + // Attached before keploy-sdk.jar: on the classpath already. + assertTrue(EmbeddedJacoco.attached(withJacoco, none)); + assertFalse(EmbeddedJacoco.attached(without, none)); + // Attached after it: only its argument names it. + assertTrue(EmbeddedJacoco.attached(without, Arrays.asList("-Xmx64m", + "-javaagent:" + other.getPath(), "-javaagent:" + jacoco.getPath() + "=output=none"))); + assertTrue(EmbeddedJacoco.attached(without, Collections.singletonList("-javaagent:" + jacoco.getPath()))); + // Another agent, a jar named as an option, a jar not there. + assertFalse(EmbeddedJacoco.attached(without, Arrays.asList("-javaagent:" + other.getPath(), + "-javaagent:" + other.getPath() + "=" + jacoco.getPath(), "-cp", jacoco.getPath(), + "-javaagent:" + new File(tmp.getRoot(), "gone.jar").getPath()))); + } + } + + @Test + public void theSdkAloneStartsJacocoOnTheAppsClassesOnly() throws Exception { + String output = runApp(new Run(SDK)); + assertTrue(output, output.contains("agent=true")); + assertTrue(output, output.contains("app=true")); + // Loaded after JaCoCo started, outside its includes=. + assertTrue(output, output.contains("library=false")); + assertTrue(output, output.contains("on the app's 1 classes alone (includes=org.example.embedded.*)")); + } + + @Test + public void aJacocoAgentAttachedAfterTheSdkIsLeftToRunAlone() throws Exception { + // The command attaches JaCoCo after keploy-sdk.jar, as the README + // showed: its premain has not run when the SDK's does, and the SDK + // must not start a second one, whose premain fails the JVM's start. + assertLeftAlone(runApp(new Run(SDK, JACOCO))); + } + + @Test + public void aJacocoAgentAfterTheSdkInJavaToolOptionsIsLeftToRunAlone() throws Exception { + Run run = new Run(SDK, JACOCO); + run.inEnv = true; + assertLeftAlone(runApp(run)); + } + + @Test + public void aJacocoAgentAttachedBeforeTheSdkIsLeftToRunAlone() throws Exception { + // k8s-proxy's order: JaCoCo, then keploy-sdk.jar, in + // JAVA_TOOL_OPTIONS. It keeps working as before, with no index read + // before the app's main. + Run run = new Run(JACOCO, SDK); + run.inEnv = true; + assertLeftAlone(runApp(run)); + } + + @Test + public void dedupTurnedOffStartsNoJacoco() throws Exception { + Run run = new Run(SDK); + run.env.put("KEPLOY_JAVA_DEDUP_DISABLED", "true"); + String output = runApp(run); + assertTrue(output, output.contains("agent=false")); + assertTrue(output, output.contains("app=false")); + assertFalse(output, output.contains("JaCoCo")); + } + + @Test + public void aJacocoThatCannotStartLeavesTheAppRunningUnmeasured() throws Exception { + // The temp dir is a file, so the jar JaCoCo runs from cannot be + // written: the app still starts, and the log says why it is not + // measured. + Run run = new Run(SDK); + run.tmpdir = tmp.newFile("not-a-dir"); + String output = runApp(run); + assertTrue(output, output.contains("agent=false")); + assertTrue(output, output.contains("app=false")); + assertTrue(output, output.contains("could not start the JaCoCo agent keploy-sdk.jar carries")); + assertFalse(output, output.contains("started JaCoCo")); + } + + private static void assertLeftAlone(String output) { + assertTrue(output, output.contains("agent=true")); + assertTrue(output, output.contains("app=true")); + // The command's JaCoCo has no includes=, so it takes in libraries. + assertTrue(output, output.contains("library=true")); + assertTrue(output, output.contains("this JVM has a JaCoCo agent attached")); + assertFalse(output, output.contains("started JaCoCo")); + assertFalse(output, output.contains("Error while instrumenting")); + // EmbeddedApp sends no START, so only a premain index read says so. + assertFalse(output, output.contains("fingerprinting the app's")); + } + + // sdkJar is a jar whose Premain-Class is the SDK's, which is on the + // classpath it is run with. + private File sdkJar() throws IOException { + File sdk = tmp.newFile("keploy-sdk-" + System.nanoTime() + ".jar"); + Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + manifest.getMainAttributes().putValue("Premain-Class", KeployDedupAgent.class.getName()); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(sdk), manifest)) { + out.putNextEntry(new ZipEntry("META-INF/keploy-test")); + out.closeEntry(); + } + return sdk; + } + + private static final String SDK = "sdk"; + private static final String JACOCO = "jacoco"; + + // Run is how runApp starts EmbeddedApp: the agents in order, on the + // command or in JAVA_TOOL_OPTIONS when inEnv, extra environment, and the + // JVM's java.io.tmpdir (a new dir when null). + private static final class Run { + final List agents; + boolean inEnv; + final Map env = new HashMap<>(); + File tmpdir; + + Run(String... agents) { + this.agents = Arrays.asList(agents); + } + } + + // runApp runs EmbeddedApp in its own JVM, as run says, from a dir of its + // own and with none of this JVM's agent or Keploy settings, and returns + // what it printed. + private String runApp(Run run) throws Exception { + String key = EmbeddedApp.class.getName().replace('.', '/') + ".class"; + File classes = tmp.newFolder(); + File copy = new File(classes, key); + assertTrue(copy.getParentFile().mkdirs()); + try (InputStream input = EmbeddedApp.class.getClassLoader().getResourceAsStream(key)) { + Files.copy(input, copy.toPath()); + } + List agents = new ArrayList<>(); + for (String agent : run.agents) { + if (agent.equals(SDK)) { + agents.add("-javaagent:" + sdkJar().getPath()); + } else { + File jacoco = tmp.newFile("jacocoagent-" + System.nanoTime() + ".jar"); + AgentJar.extractTo(jacoco); + agents.add("-javaagent:" + jacoco.getPath() + "=output=none"); + } + } + File tmpdir = run.tmpdir != null ? run.tmpdir : tmp.newFolder(); + List command = new ArrayList<>(); + command.add(new File(System.getProperty("java.home"), "bin/java").getPath()); + command.add("-Djava.io.tmpdir=" + tmpdir.getPath()); + if (!run.inEnv) { + command.addAll(agents); + } + command.add("-cp"); + command.add(System.getProperty("java.class.path")); + command.add(EmbeddedApp.class.getName()); + File log = tmp.newFile(); + ProcessBuilder builder = new ProcessBuilder(command).directory(tmp.newFolder()) + .redirectErrorStream(true).redirectOutput(log); + Map env = builder.environment(); + env.keySet().removeIf(name -> name.startsWith("KEPLOY_") || name.equals("JAVA_TOOL_OPTIONS") + || name.equals("JDK_JAVA_OPTIONS") || name.equals("_JAVA_OPTIONS")); + if (run.inEnv) { + env.put("JAVA_TOOL_OPTIONS", String.join(" ", agents)); + } + env.put("KEPLOY_JAVA_CLASS_DIRS", classes.getPath()); + // TCP: no socket file in /tmp. Nothing listens there; the SDK dials + // in the background, and the app runs on. + env.put("KEPLOY_COVERAGE_ENDPOINT", "127.0.0.1:1"); + env.putAll(run.env); + Process process = builder.start(); + boolean exited = process.waitFor(60, TimeUnit.SECONDS); + if (!exited) { + process.destroyForcibly().waitFor(10, TimeUnit.SECONDS); + } + String output = new String(Files.readAllBytes(log.toPath()), StandardCharsets.UTF_8); + assertTrue(output, exited); + assertEquals(output, 0, process.exitValue()); + return output; + } +} diff --git a/keploy-sdk/src/test/java/org/example/embedded/EmbeddedApp.java b/keploy-sdk/src/test/java/org/example/embedded/EmbeddedApp.java new file mode 100644 index 00000000..8f6e5a01 --- /dev/null +++ b/keploy-sdk/src/test/java/org/example/embedded/EmbeddedApp.java @@ -0,0 +1,37 @@ +package org.example.embedded; + +import java.lang.reflect.Method; + +/** + * EmbeddedApp is the app of EmbeddedJacocoTest's own JVMs: it says whether a + * JaCoCo agent runs (agent=false when none was attached or started), and + * whether JaCoCo instrumented its own class and a library class it loads only + * now, after every agent's premain. + */ +public final class EmbeddedApp { + + private EmbeddedApp() { + } + + public static void main(String[] args) throws Exception { + Object agent; + try { + agent = Class.forName("org.jacoco.agent.rt.RT").getMethod("getAgent").invoke(null); + } catch (ClassNotFoundException e) { + // No JaCoCo agent was attached or started. + agent = null; + } + System.out.println("agent=" + (agent != null)); + System.out.println("app=" + instrumented(EmbeddedApp.class)); + System.out.println("library=" + instrumented(Class.forName("org.junit.runner.Description"))); + } + + private static boolean instrumented(Class type) { + for (Method method : type.getDeclaredMethods()) { + if (method.getName().equals("$jacocoInit")) { + return true; + } + } + return false; + } +} diff --git a/scripts/smoke-javaagent.sh b/scripts/smoke-javaagent.sh index fd00a461..af75bdd3 100755 --- a/scripts/smoke-javaagent.sh +++ b/scripts/smoke-javaagent.sh @@ -245,6 +245,35 @@ jar cfm "${work_dir}/jar/plain-app.jar" "${work_dir}/jar/MANIFEST.MF" -C "${work -jar plain-app.jar "plain-jar" ) +# keploy-sdk.jar alone: it starts the JaCoCo agent it carries, with includes= +# the app's classes, so the command attaches no jacocoagent.jar. Its log +# must say it did, or a JaCoCo found elsewhere could pass these cases. +java_with_sdk_only() { + local mode="${1:?mode required}" + shift + java -javaagent:"${agent_jar}" "$@" 2>"${work_dir}/${mode}.err" || { cat "${work_dir}/${mode}.err" >&2; exit 1; } + if ! grep -q "started JaCoCo .* on the app's .* classes alone (includes=" "${work_dir}/${mode}.err"; then + cat "${work_dir}/${mode}.err" >&2 + echo "${mode}: keploy-sdk.jar did not start its JaCoCo agent on the app's classes" >&2 + exit 1 + fi +} + +( + cd "${work_dir}/maven" + run_smoke "sdk-only-maven-classes" \ + java_with_sdk_only "sdk-only-maven-classes" \ + -cp "target/classes:${agent_jar}" \ + smoke.SmokeHarness "sdk-only-maven-classes" +) + +( + cd "${work_dir}/jar" + run_smoke "sdk-only-plain-jar" \ + java_with_sdk_only "sdk-only-plain-jar" \ + -jar plain-app.jar "sdk-only-plain-jar" +) + mkdir -p "${work_dir}/boot-root/BOOT-INF/classes" "${work_dir}/war-root/WEB-INF/classes" cp -R "${work_dir}/classes/." "${work_dir}/boot-root/BOOT-INF/classes/" cp -R "${work_dir}/classes/." "${work_dir}/war-root/WEB-INF/classes/" From 36fb8769ad64df24e2c41c30f9b221e31bb8f1d2 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Thu, 19 Nov 2026 23:25:29 +0530 Subject: [PATCH 3/5] feat(dedup): tell Keploy how the app's classes were found Keploy could not tell whether dedup ran on the app's own classes or fell back to the whole jar or classpath: the SDK said so only in the app's log. When it fell back, dedup kept far more tests than it should, and the Keploy run looked normal. Before the first test's coverage, the SDK now sends Keploy one INDEX record on /tmp/coverage_data.sock. It is a JSON object on its own connection, framed as each test's coverage record is, and it starts with {"kind":"index",. It holds: - source: override (KEPLOY_JAVA_CLASS_DIRS), derived (from the launch), whole-jar, classpath (the app's classes could not be told from its libraries) or none; - launch: how the JVM was launched, or why the app's classes could not be told apart; - roots: the dirs and jars the classes were read from, up to 64 (rootCount gives the full count); in a Spring Boot jar these are app.jar!BOOT-INF/classes and app.jar!BOOT-INF/lib/.jar; - packages: the app's package root when it was derived and has two parts or more, else empty; - classes: how many classes were indexed; - build: "jv1-" and the first 32 hex of the sha256 over the sorted, unsigned, de-duplicated JaCoCo class ids (CRC64 of each class's bytes, 16 lowercase hex, joined by newlines), the id Keploy gives the same build; "" when no class was found; - includes and jacoco: the includes= of the JaCoCo agent keploy-sdk.jar started (embedded), or that the command attached its own (attached), that none runs (none), or that the SDK was not started from premain (unchecked). It carries no class list. The worker sends it right after the control socket is bound and before it reads any command, when the data socket's file is there (Keploy's receiver makes it); a START sent meanwhile waits in the socket's backlog. With no socket file there it sends nothing and does not read the index for it, so a JVM that runs with no Keploy does no extra work. A file left by a Keploy that died makes that send fail at once; it is tried again before the next test's coverage. The SDK sends it again before the next test's coverage when the receiver on the data socket is a new one (a new run, a docker-compose test set's receiver, a restarted Keploy, or none there at the first send). It tells receivers apart by the socket file's identity and the time it was made. A send that fails never fails the test's END. source is override only when the dirs and jars KEPLOY_JAVA_CLASS_DIRS names hold a class. When they hold none, the SDK now says so at WARNING and reports what it found instead, for example the build output under the working dir, as derived. A launch that lists more than five dirs now says how many more there are. Limits of this change: - On the TCP transport used in Kubernetes no INDEX record is sent. - The order is the send order: Keploy reads each connection on its own, so it may handle a test's record before the INDEX sent just ahead of it. A receiver replaced while the SDK sends can get the INDEX twice. - A Keploy that does not know the record stores it as a coverage record with an empty test id, then logs "Received malformed test ID, skipping dedup file update" at ERROR, once for each start of the app and again for each new receiver: on `keploy test`, which starts the app again for every test set, and on docker compose alike, that is once per test set. Its dedupData.yaml does not change. In a run where no test's coverage arrives, for example when the record says no class was found, that Keploy also logs that dedupData.yaml was generated although it writes none. Tests: IndexReportTest checks the build id against the value Keploy's own test gives for the same ids, in any order and repeated, and the record's shape. AppIndexTest checks the record for derived dirs, the build output under the working dir, an override, an override with no class, a Spring Boot jar with no source tree, the whole jar, the whole classpath, and no class, and that a JVM with no receiver does not read its index for it. ControlSocketTest checks that the record reaches a receiver once before any command, that a receiver bound again on the same path gets it before its first coverage record, and that a socket file's new time, or another file with the same time, tells a new receiver apart. Three forked EmbeddedJacocoTest rows check includes= and jacoco for the JaCoCo agent the SDK started, for one the command attached, and none when it could not start. scripts/smoke-javaagent.sh, which CI runs on JDK 8, 17 and 21, now takes the INDEX record as the first on the data socket, then the test's coverage, and checks its source and jacoco for each launch. Signed-off-by: slayerjain --- README.md | 2 + .../java/io/keploy/dedup/IndexReport.java | 164 +++++++++++++++++ .../io/keploy/dedup/KeployDedupAgent.java | 172 ++++++++++++++++-- .../java/io/keploy/dedup/AppIndexTest.java | 124 +++++++++++++ .../io/keploy/dedup/ControlSocketTest.java | 81 ++++++++- .../io/keploy/dedup/EmbeddedJacocoTest.java | 18 +- .../java/io/keploy/dedup/IndexReportTest.java | 95 ++++++++++ .../org/example/embedded/EmbeddedApp.java | 14 +- scripts/smoke-javaagent.sh | 52 ++++-- 9 files changed, 694 insertions(+), 28 deletions(-) create mode 100644 keploy-sdk/src/main/java/io/keploy/dedup/IndexReport.java create mode 100644 keploy-sdk/src/test/java/io/keploy/dedup/IndexReportTest.java diff --git a/README.md b/README.md index 5cef3f38..540f1097 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ Keploy Enterprise drives dynamic dedup per testcase. Coverage is collected at per-testcase granularity, not process granularity. +Before the first testcase's coverage, the agent sends Enterprise one INDEX record on `/tmp/coverage_data.sock`: a JSON object starting with `{"kind":"index",` that says how it found the application's own classes (`source`: `override` when the dirs and jars in `KEPLOY_JAVA_CLASS_DIRS` held them, `derived` from the launch or from `target/classes` or `build/classes/java/main` under the working directory, `whole-jar` or `classpath` when it could not tell them from the libraries, `none`), what it found them from in words (`launch`), the dirs and jars it read them from (`roots`, at most 64, and `rootCount`), the package roots it kept (`packages`), how many classes it indexed, the build id of those classes (`jv1-` and 32 hex), which JaCoCo agent measures them (`jacoco`: `embedded`, the one keploy-sdk.jar starts, `attached` by the command, `none`, or `unchecked` when the agent was not started as a `-javaagent`), and that embedded agent's `includes=`. It carries no class list. It is sent as the agent starts when that path's socket file is there (Enterprise's receiver makes it), and otherwise, or when that send fails, right before the first testcase's coverage; a receiver Enterprise binds anew on that path gets it again before its first record. On the TCP transport used in Kubernetes no INDEX record is sent. + The agent binds `/tmp/coverage_control.sock` before the application's `main` runs, so the socket is listening before the application can answer a health check. It answers `ACK` to each command it took, and `IGNORED` to an `END` that no `START` for that testcase came before: nothing is published for it, and Enterprise keeps that testcase out of the comparison. On the TCP transport used in Kubernetes the same `END` is answered with `IGNORED ` and then `ACK`. ## How to Use diff --git a/keploy-sdk/src/main/java/io/keploy/dedup/IndexReport.java b/keploy-sdk/src/main/java/io/keploy/dedup/IndexReport.java new file mode 100644 index 00000000..32530bf6 --- /dev/null +++ b/keploy-sdk/src/main/java/io/keploy/dedup/IndexReport.java @@ -0,0 +1,164 @@ +package io.keploy.dedup; + +import com.google.gson.Gson; +import org.jacoco.core.internal.data.CRC64; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.TreeSet; + +/** + * IndexReport is the INDEX record: what the SDK took as the app's classes, + * sent to Keploy on the data socket once for each receiver there, before + * the first test's coverage it gets (KeployDedupAgent's CoveragePublisher). + * Keploy says from it how the app's classes were found, and when they were + * not (the whole jar, the whole classpath, none), and keys the build table + * by its build id. + *

+ * It is one JSON object on its own connection, as each test's coverage + * record is, told apart by {@code "kind":"index"}. It carries no class list: + * the build table carries every class. + */ +final class IndexReport { + + static final String KIND = "index"; + static final int VERSION = 1; + // A report names at most this many roots, and says how many there were. + static final int MAX_ROOTS = 64; + + // How the index was found. + static final String SOURCE_OVERRIDE = "override"; + static final String SOURCE_DERIVED = "derived"; + static final String SOURCE_WHOLE_JAR = "whole-jar"; + static final String SOURCE_CLASSPATH = "classpath"; + static final String SOURCE_NONE = "none"; + + // What measures the classes: the JaCoCo agent keploy-sdk.jar started + // (includes= the index), one the command attached (every class its own + // options take in), none (premain started none: no class, or it failed), + // or unchecked (the SDK was not started by premain, so it starts none and + // looked for none). + static final String JACOCO_EMBEDDED = "embedded"; + static final String JACOCO_ATTACHED = "attached"; + static final String JACOCO_NONE = "none"; + static final String JACOCO_UNCHECKED = "unchecked"; + + private static final Gson GSON = new Gson(); + + // Serialized in this order, by name. + final String kind = KIND; + final int v = VERSION; + // "jv1-" + 32 hex (buildId), or "" when the index holds no class. + final String build; + final String source; + // How the JVM was launched, or why the app's classes could not be told + // apart; for an override, the dirs and jars it named. + final String launch; + final List roots; + final int rootCount; + // The app's package roots ("org.apache.fineract"), when derived. + final List packages; + final int classes; + // includes= of the JaCoCo agent the SDK started, or "". + final String includes; + final String jacoco; + + IndexReport(String build, String source, String launch, Collection roots, List packages, + int classes, String includes, String jacoco) { + this.build = build; + this.source = source; + this.launch = launch; + List sorted = new ArrayList<>(new TreeSet<>(roots)); + this.rootCount = sorted.size(); + this.roots = sorted.size() > MAX_ROOTS ? new ArrayList<>(sorted.subList(0, MAX_ROOTS)) : sorted; + this.packages = packages; + this.classes = classes; + this.includes = includes; + this.jacoco = jacoco; + } + + String toJson() { + return GSON.toJson(this); + } + + /** + * classId is the id JaCoCo gives a class: the CRC64 of the bytes the + * index read. It equals the id in JaCoCo's ExecutionData only when JaCoCo + * saw those same bytes: not for a class another agent transformed first, + * nor for a multi-release jar's META-INF/versions copy the JVM loads in + * place of the base entry the index reads. + */ + static long classId(byte[] bytes) { + return CRC64.classId(bytes); + } + + /** + * buildId is the build's id from its classes' JaCoCo ids: "jv1-" and the + * first 32 hex of the sha256 over the ids sorted as unsigned numbers, + * de-duplicated, each as 16 lowercase hex, joined by newlines. Keploy's + * codecoverage.JavaBuildID computes the same from the same ids. "" when + * there is no id. + */ + static String buildId(long[] ids) { + if (ids.length == 0) { + return ""; + } + long[] sorted = Arrays.copyOf(ids, ids.length); + // Unsigned order: flipping the sign bit maps it onto signed order. + for (int i = 0; i < sorted.length; i++) { + sorted[i] ^= Long.MIN_VALUE; + } + Arrays.sort(sorted); + StringBuilder joined = new StringBuilder(sorted.length * 17); + for (int i = 0; i < sorted.length; i++) { + if (i > 0 && sorted[i] == sorted[i - 1]) { + continue; + } + if (joined.length() > 0) { + joined.append('\n'); + } + String hex = Long.toHexString(sorted[i] ^ Long.MIN_VALUE); + for (int pad = hex.length(); pad < 16; pad++) { + joined.append('0'); + } + joined.append(hex); + } + byte[] sum; + try { + sum = MessageDigest.getInstance("SHA-256").digest(joined.toString().getBytes(StandardCharsets.US_ASCII)); + } catch (NoSuchAlgorithmException e) { + // Every Java platform has SHA-256. + throw new IllegalStateException(e); + } + StringBuilder id = new StringBuilder("jv1-"); + for (int i = 0; i < 16; i++) { + id.append(Character.forDigit((sum[i] >> 4) & 0xf, 16)).append(Character.forDigit(sum[i] & 0xf, 16)); + } + return id.toString(); + } + + /** + * rootOf is the dir or jar a class was read from: its location + * ("/app/classes/org/acme/Foo.class", "/app/app.jar!BOOT-INF/lib/x.jar!org/acme/Foo.class") + * without the class's own path, or null when the location does not end + * with it. + */ + static String rootOf(String location, String className) { + String suffix = className + ".class"; + if (!location.endsWith(suffix) || location.length() == suffix.length()) { + return null; + } + String root = location.substring(0, location.length() - suffix.length()); + char last = root.charAt(root.length() - 1); + if (last != '/' && last != '!') { + return null; + } + root = root.substring(0, root.length() - 1); + return root.isEmpty() ? "/" : root; + } +} diff --git a/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java b/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java index a61fe5f3..26a442a5 100644 --- a/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java +++ b/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java @@ -33,6 +33,8 @@ import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Enumeration; @@ -123,6 +125,8 @@ static String encodeBytecodeFrame(String buildTag, String manifestJson, byte[] z // socket file to the first one's close. Before the first start() it is a // thread never started, which join() does not wait for. private static volatile Thread coverageThread = new Thread("keploy-java-dedup-none"); + // The index the running agent was started on, for indexReport. + private static volatile CoverageIndex startedIndex; // How long stop(), and a start() after it, wait for the worker's thread. // Not final: the tests shorten it. static volatile long workerStopWaitMillis = 5000; @@ -179,6 +183,7 @@ private static boolean start(CoverageIndex index, Instrumentation instrumentatio return false; } + startedIndex = index; CoverageCollector collector = new CoverageCollector( new JacocoClient(resolveHost(), resolvePort()), index); @@ -223,7 +228,7 @@ private static boolean start(CoverageIndex index, Instrumentation instrumentatio // up after the app answered and Keploy had fired its first tests, // whose START and END then reached no one: their coverage was lost. CommandServer server = new CommandServer(collector, - new CoveragePublisher(new File(dataSocketPath)), new File(controlSocketPath)); + new CoveragePublisher(new File(dataSocketPath), index), new File(controlSocketPath)); try { server.bind(); } catch (Throwable t) { @@ -257,8 +262,10 @@ private static boolean start(CoverageIndex index, Instrumentation instrumentatio // bytes, which the first START read before. private static void startJacoco(Instrumentation instrumentation, CoverageIndex index) { try { + index.jacoco = IndexReport.JACOCO_NONE; if (EmbeddedJacoco.attached(ClassLoader.getSystemClassLoader(), ManagementFactory.getRuntimeMXBean().getInputArguments())) { + index.jacoco = IndexReport.JACOCO_ATTACHED; log(Level.INFO, "Keploy dedup: this JVM has a JaCoCo agent attached, so Keploy reads coverage" + " from it, and it instruments every class its own options take in. Attach only" + " keploy-sdk.jar to have JaCoCo instrument the app's own classes alone", null); @@ -276,6 +283,8 @@ private static void startJacoco(Instrumentation instrumentation, CoverageIndex i return; } String version = EmbeddedJacoco.start(instrumentation, EmbeddedJacoco.options(includes)); + index.includes = includes; + index.jacoco = IndexReport.JACOCO_EMBEDDED; log(Level.INFO, "Keploy dedup: started JaCoCo " + version + (includes.isEmpty() ? " on every class" : " on the app's " + names.size() + " classes alone (includes=" @@ -289,6 +298,13 @@ private static void startJacoco(Instrumentation instrumentation, CoverageIndex i } } + // indexReport is the INDEX report of the index the agent was last started + // on, as sent to Keploy, or null before any start (tests). + static String indexReport() { + CoverageIndex index = startedIndex; + return index == null ? null : index.report().toJson(); + } + /** * Returns whether the background control socket listener is active. * @@ -430,6 +446,22 @@ private static Object fileKey(File file) { } } + // receiverKey tells apart the receivers Keploy binds, one after another, + // on the data socket's path: the socket file's identity and the time it + // was made, as a file system can give a new file the inode of one just + // removed (and nothing but a new bind changes a socket file's times). + // Where the platform has no file key, the time alone. Null when there is + // no file: no receiver to reach. + static Object receiverKey(File file) { + try { + BasicFileAttributes attributes = Files.readAttributes(file.toPath(), BasicFileAttributes.class, + LinkOption.NOFOLLOW_LINKS); + return Arrays.asList(attributes.fileKey(), attributes.lastModifiedTime()); + } catch (IOException e) { + return null; + } + } + private static void deleteSocketFile(File file) { if (file.exists() && !file.delete()) { log(Level.FINE, "Failed to delete socket file " + file.getAbsolutePath(), null); @@ -527,6 +559,11 @@ void bind() throws IOException { @Override public void run() { try (AFUNIXServerSocket localServer = serverSocket) { + // Before any command, when Keploy's receiver is already + // bound: a START sent meanwhile waits in the socket's + // backlog. Off premain, as it reads the index's bytes when + // nothing has yet. + publisher.sendIndex(); while (running.get()) { Socket accepted; try { @@ -1440,6 +1477,46 @@ private List entries() { // came from. Set by loadEntries, before entries() returns. private volatile boolean appOnly; private volatile String source = ""; + // For the INDEX report: how the index was found (IndexReport's + // SOURCE_*), how the JVM was launched or why the app's classes could + // not be told apart, and the app's package roots when derived. Set by + // loadEntries, before entries() returns. + private volatile String sourceKind = IndexReport.SOURCE_NONE; + private volatile String launch = ""; + private volatile List packages = Collections.emptyList(); + // What measures the classes, and the includes= of the JaCoCo agent + // the SDK started (IndexReport's JACOCO_*). Set by premain's + // startJacoco before the worker that sends the report starts. + private volatile String jacoco = IndexReport.JACOCO_UNCHECKED; + private volatile String includes = ""; + private volatile IndexReport report; + + // report is the INDEX report of this index: computed once, on the + // first call, from entries() (so it reads the index if nothing has). + private IndexReport report() { + IndexReport cached = report; + if (cached != null) { + return cached; + } + synchronized (lock) { + if (report == null) { + List all = entries(); + long[] ids = new long[all.size()]; + Set roots = new LinkedHashSet<>(); + for (int i = 0; i < ids.length; i++) { + ClassEntry entry = all.get(i); + ids[i] = IndexReport.classId(entry.bytes); + String root = IndexReport.rootOf(entry.location, entry.className); + if (root != null) { + roots.add(root); + } + } + report = new IndexReport(IndexReport.buildId(ids), sourceKind, launch, roots, packages, + all.size(), includes, jacoco); + } + return report; + } + } // isAppOnly says whether the index holds only the app's own classes. // When it does not (a whole jar or the whole classpath), fingerprints @@ -1451,18 +1528,35 @@ private boolean isAppOnly() { private List loadEntries() { LinkedHashMap collected = new LinkedHashMap<>(); + List configuredRoots = configuredClassRoots(); List applicationRoots = applicationRoots(); List executableArchiveRoots = Collections.emptyList(); List classpathRoots = Collections.emptyList(); - scanRoots(applicationRoots, collected); + // KEPLOY_JAVA_CLASS_DIRS's own roots first: the index is the + // override's only when they hold a class, not when the build + // output under the working dir does. + scanRoots(configuredRoots, collected); + boolean overridden = !collected.isEmpty(); + if (!configuredRoots.isEmpty() && !overridden) { + log(Level.WARNING, "Keploy dedup found no class in KEPLOY_JAVA_CLASS_DIRS " + + summarizeFiles(configuredRoots, 5) + ", so it looks for the app's classes itself." + + " Set it to the app's class dirs and jars", null); + } + List defaultRoots = new ArrayList<>(applicationRoots); + defaultRoots.removeAll(configuredRoots); + scanRoots(defaultRoots, collected); List foundRoots = new ArrayList<>(); - for (File root : applicationRoots) { + // An override that held no class is not where the classes came from. + for (File root : overridden ? applicationRoots : defaultRoots) { if (root.exists()) { foundRoots.add(root); } } String from = "the class dirs " + summarizeFiles(foundRoots, 5); + String kind = overridden ? IndexReport.SOURCE_OVERRIDE : IndexReport.SOURCE_DERIVED; + String how = from; + List pkgRoots = Collections.emptyList(); boolean onlyApp = true; // why says why the index is not the app's own classes alone. String why = ""; @@ -1478,20 +1572,31 @@ private List loadEntries() { from = derived.root == null ? derived.launch : "the package " + derived.root + " (" + derived.launch + ")"; why = derived.failed ? derived.launch : "no class found from " + from; + kind = IndexReport.SOURCE_DERIVED; + how = derived.launch; + pkgRoots = derived.root == null ? Collections.emptyList() + : Collections.singletonList(derived.root); } if (collected.isEmpty()) { onlyApp = false; executableArchiveRoots = executableArchiveRoots(); scanRoots(executableArchiveRoots, collected); from = "the whole jar " + summarizeFiles(executableArchiveRoots, 5); + kind = IndexReport.SOURCE_WHOLE_JAR; + how = why; + pkgRoots = Collections.emptyList(); } if (collected.isEmpty() && isClasspathFallbackEnabled()) { classpathRoots = classpathRoots(); scanRoots(classpathRoots, collected); from = "the whole classpath"; + kind = IndexReport.SOURCE_CLASSPATH; } appOnly = onlyApp; source = collected.isEmpty() ? "no class" : from; + sourceKind = collected.isEmpty() ? IndexReport.SOURCE_NONE : kind; + launch = how; + packages = collected.isEmpty() ? Collections.emptyList() : pkgRoots; if (collected.isEmpty()) { // Said at WARNING: no test's coverage is measured at all. log(Level.WARNING, "Keploy dedup found none of the app's classes (" + why + "), nor any class" @@ -1530,20 +1635,24 @@ public int compare(ClassEntry left, ClassEntry right) { return sorted; } - private String summarizeFiles(Iterable values, int limit) { + // summarizeFiles lists the first limit of values, and how many more + // there are. + private String summarizeFiles(Collection values, int limit) { List sample = new ArrayList<>(); for (File value : values) { - sample.add(value.getPath()); - if (sample.size() >= limit) { + if (sample.size() == limit) { + sample.add("and " + (values.size() - limit) + " more"); break; } + sample.add(value.getPath()); } return sample.toString(); } - private List applicationRoots() { + // configuredClassRoots is the dirs and jars KEPLOY_JAVA_CLASS_DIRS + // names, in its order. + private List configuredClassRoots() { LinkedHashSet roots = new LinkedHashSet<>(); - String configured = envOrProperty("KEPLOY_JAVA_CLASS_DIRS", "keploy.java.class.dirs", ""); if (!configured.trim().isEmpty()) { for (String part : configuredRoots(configured)) { @@ -1552,6 +1661,11 @@ private List applicationRoots() { } } } + return new ArrayList<>(roots); + } + + private List applicationRoots() { + LinkedHashSet roots = new LinkedHashSet<>(configuredClassRoots()); roots.add(new File(System.getProperty("user.dir"), "target/classes")); roots.add(new File(System.getProperty("user.dir"), "build/classes/java/main")); @@ -1723,15 +1837,49 @@ private String classNameFromKey(String key) { private static final class CoveragePublisher { private final File socketFile; - - private CoveragePublisher(File socketFile) { + private final CoverageIndex index; + // The receiver (receiverKey) the INDEX report last reached, null + // before the first: one Keploy binds anew on the data socket's path (a + // new run, a docker-compose test set, a restarted Keploy) gets it + // again before its first record. + private Object indexSentTo; + + private CoveragePublisher(File socketFile, CoverageIndex index) { this.socketFile = socketFile; + this.index = index; + } + + // sendIndex sends the INDEX report unless the receiver on the data + // socket has it: once right after the control socket is bound, and + // again before a test's coverage when the receiver there is new. With + // no socket file there is no receiver to reach, and the report (which + // reads every indexed class) is not made: a JVM that runs with no + // Keploy never reads its index for it. A send that fails is tried + // again before the next test's coverage. It never fails the caller: + // Keploy then says only that it has no INDEX report. + synchronized void sendIndex() { + try { + Object key = receiverKey(socketFile); + if (key == null || key.equals(indexSentTo)) { + return; + } + send(index.report().toJson().getBytes(StandardCharsets.UTF_8)); + indexSentTo = key; + } catch (Throwable t) { + // Throwable: an Error here (the class ids' JaCoCo classes + // missing) must not end the worker, nor fail a test's END. + log(Level.FINE, "Could not send the Java dedup INDEX report to " + socketFile.getAbsolutePath(), t); + } } private void publish(String testId, Map> executedProbesByClass) throws IOException { - byte[] payload = GSON.toJson(new DedupPayload(testId, executedProbesByClass)) - .getBytes(StandardCharsets.UTF_8); + sendIndex(); + send(GSON.toJson(new DedupPayload(testId, executedProbesByClass)).getBytes(StandardCharsets.UTF_8)); + } + // send writes payload on a connection of its own to the data socket: + // one record per connection, ended by its close. + private void send(byte[] payload) throws IOException { try (AFUNIXSocket socket = AFUNIXSocket.newInstance()) { socket.connect(AFUNIXSocketAddress.of(socketFile), SOCKET_TIMEOUT_MILLIS); socket.setSoTimeout(SOCKET_TIMEOUT_MILLIS); diff --git a/keploy-sdk/src/test/java/io/keploy/dedup/AppIndexTest.java b/keploy-sdk/src/test/java/io/keploy/dedup/AppIndexTest.java index 5e48ca4e..0c3be885 100644 --- a/keploy-sdk/src/test/java/io/keploy/dedup/AppIndexTest.java +++ b/keploy-sdk/src/test/java/io/keploy/dedup/AppIndexTest.java @@ -10,6 +10,10 @@ import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -19,6 +23,10 @@ import java.util.logging.LogRecord; import java.util.logging.Logger; import java.util.zip.ZipEntry; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.jacoco.core.internal.data.CRC64; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -110,6 +118,8 @@ public void derivedAppClassesAreFingerprintedAndWarmedUp() throws Exception { assertLogged("INFO Keploy dedup: fingerprinting the app's 2 classes, from the package com.acme.shop" + " (java -cp ... com.acme.shop.Main)"); + assertIndex("derived", "java -cp ... com.acme.shop.Main", "[\"com.acme.shop\"]", 2, + classes.getPath()); // The fake classes cannot be loaded: the warmup tried them both. assertLogged("INFO Keploy dedup: warmed up application classes (initialized=0, skipped=2)"); assertNotLogged("no warmup"); @@ -126,7 +136,26 @@ public void configuredClassDirsAreFingerprintedAndWarmedUp() throws Exception { startAndSendStart(); assertLogged("INFO Keploy dedup: fingerprinting the app's 1 classes, from the class dirs [" + classes.getPath() + "]."); + assertIndex("override", "the class dirs [" + classes.getPath() + "]", "[]", 1, classes.getPath()); assertLogged("INFO Keploy dedup: warmed up application classes (initialized=0, skipped=1)"); + assertNotLogged("found no class in KEPLOY_JAVA_CLASS_DIRS"); + } + + @Test + public void manyClassDirsAreSaidWithHowManyMore() throws Exception { + List dirs = new ArrayList<>(); + for (int i = 0; i < 7; i++) { + dirs.add(tmp.newFolder("module" + i).getPath()); + } + write(new File(dirs.get(6), "Main.class")); + System.setProperty("keploy.java.class.dirs", String.join(File.pathSeparator, dirs)); + System.setProperty("java.class.path", dirs.get(6)); + System.setProperty("sun.java.command", "Main"); + + startAndSendStart(); + + assertIndex("override", "the class dirs [" + String.join(", ", dirs.subList(0, 5)) + ", and 2 more]", "[]", + 1, dirs.get(6)); } @Test @@ -201,6 +230,9 @@ public void aSpringBootJarWithNoSourceTreeIsDerivedByClassNames() throws Excepti assertLogged("INFO Keploy dedup: fingerprinting the app's 3 classes, from the package com.acme.shop" + " (java -jar " + boot.getPath() + ")"); + // No source, no git: the roots are inside the jar. + assertIndex("derived", "java -jar " + boot.getPath(), "[\"com.acme.shop\"]", 3, + boot.getAbsolutePath() + "!BOOT-INF/classes", boot.getAbsolutePath() + "!BOOT-INF/lib/shop-model.jar"); } @Test @@ -246,6 +278,8 @@ public void wholeClasspathIsSaidAndNeverWarmedUp() throws Exception { assertLogged("WARNING Keploy dedup could not tell the app's own classes from its libraries (java -cp ..." + " org.apache.catalina.startup.Bootstrap, the launcher of org.apache.catalina.startup, not the" + " app), so it fingerprints all 2 classes of the whole classpath"); + assertIndex("classpath", "java -cp ... org.apache.catalina.startup.Bootstrap, the launcher of" + + " org.apache.catalina.startup, not the app", "[]", 2, bootstrap.getAbsolutePath(), classes.getPath()); assertLogged("INFO Keploy dedup: no warmup, as the index is not the app's own classes alone" + " (from the whole classpath)"); assertNotLogged("warmed up"); @@ -266,6 +300,8 @@ public void nothingFoundIsSaidAtWarning() throws Exception { + " org.apache.catalina.startup.Bootstrap, the launcher of org.apache.catalina.startup, not the" + " app), nor any class in the jar, and KEPLOY_JAVA_CLASSPATH_FALLBACK=false keeps it off the" + " classpath, so no test's coverage is measured. Set KEPLOY_JAVA_CLASS_DIRS"); + assertIndex("none", "java -cp ... org.apache.catalina.startup.Bootstrap, the launcher of" + + " org.apache.catalina.startup, not the app", "[]", 0); assertNotLogged("fingerprint"); assertNotLogged("warmed up"); } @@ -283,6 +319,8 @@ public void aMainClassOffTheClasspathFallsBackAndSaysWhere() throws Exception { + " com.acme.shop.Main, but no dir or jar on the classpath holds com/acme/shop/Main.class), so it" + " fingerprints all 1 classes of the whole jar [" + jar.getPath() + "]"); assertLogged("INFO Keploy dedup: no warmup"); + assertIndex("whole-jar", "java -cp ... com.acme.shop.Main, but no dir or jar on the classpath holds" + + " com/acme/shop/Main.class", "[]", 1, jar.getPath()); } @Test @@ -299,6 +337,57 @@ public void aRootWithNoClassFallsBackAndSaysWhere() throws Exception { assertLogged("WARNING Keploy dedup could not tell the app's own classes from its libraries (no class found" + " from the package com.acme.shop (java -cp ... com.acme.shop.MainTest)), so it fingerprints all 1" + " classes of the whole classpath"); + assertIndex("classpath", "no class found from the package com.acme.shop (java -cp ..." + + " com.acme.shop.MainTest)", "[]", 1, other.getPath()); + } + + @Test + public void theBuildOutputUnderTheWorkingDirIsDerived() throws Exception { + // mvn's target/classes under the dir the app runs from, no override. + File classes = new File(System.getProperty("user.dir"), "target/classes"); + write(new File(classes, "com/acme/shop/Main.class")); + System.setProperty("java.class.path", classes.getPath()); + System.setProperty("sun.java.command", "com.acme.shop.Main"); + + startAndSendStart(); + + assertIndex("derived", "the class dirs [" + classes.getPath() + "]", "[]", 1, classes.getPath()); + assertNotLogged("found no class in KEPLOY_JAVA_CLASS_DIRS"); + } + + @Test + public void anOverrideWithNoClassIsSaidAndIsNotTheSource() throws Exception { + // KEPLOY_JAVA_CLASS_DIRS names a dir with no class; the build output + // under the working dir holds the app. + File wrong = tmp.newFolder("wrong"); + File classes = new File(System.getProperty("user.dir"), "target/classes"); + write(new File(classes, "com/acme/shop/Main.class")); + System.setProperty("keploy.java.class.dirs", wrong.getPath()); + System.setProperty("java.class.path", classes.getPath()); + System.setProperty("sun.java.command", "com.acme.shop.Main"); + + startAndSendStart(); + + assertLogged("WARNING Keploy dedup found no class in KEPLOY_JAVA_CLASS_DIRS [" + wrong.getPath() + + "], so it looks for the app's classes itself. Set it to the app's class dirs and jars"); + assertIndex("derived", "the class dirs [" + classes.getPath() + "]", "[]", 1, classes.getPath()); + } + + @Test + public void aJvmWithNoReceiverNeverReadsItsIndexForTheReport() throws Exception { + // Keploy has bound no data socket: the INDEX report sent at the + // control socket's bind reaches no one, so it is not made, and the + // index (every class's bytes) is not read for it. + File classes = tmp.newFolder("app", "classes"); + write(new File(classes, "com/acme/shop/Main.class")); + System.setProperty("java.class.path", classes.getPath()); + System.setProperty("sun.java.command", "com.acme.shop.Main"); + + assertTrue(KeployDedupAgent.start()); + // Answered after the send at the bind: the worker runs it before its + // first accept. + assertEquals("IGNORED proto=2\n", send("END test-set-0/test-0")); + assertNotLogged("fingerprint"); } private void startAndSendStart() throws IOException { @@ -306,6 +395,41 @@ private void startAndSendStart() throws IOException { assertEquals("ACK proto=2\n", send("START test-set-0/test-1")); } + // assertIndex checks the INDEX report of the index the agent started on. + // Every fake class here is the one byte 1, so a build has one class id. + private static void assertIndex(String source, String launch, String packages, int classes, String... roots) { + JsonObject index = JsonParser.parseString(KeployDedupAgent.indexReport()).getAsJsonObject(); + assertEquals(index.toString(), source, index.get("source").getAsString()); + assertEquals(index.toString(), launch, index.get("launch").getAsString()); + assertEquals(index.toString(), packages, index.get("packages").toString()); + assertEquals(index.toString(), classes, index.get("classes").getAsInt()); + List want = new ArrayList<>(Arrays.asList(roots)); + Collections.sort(want); + assertEquals(index.toString(), new Gson().toJson(want), index.get("roots").toString()); + assertEquals(index.toString(), want.size(), index.get("rootCount").getAsInt()); + String build = ""; + if (classes > 0) { + String hex = String.format("%016x", CRC64.classId(new byte[] {1})); + build = "jv1-" + sha256(hex).substring(0, 32); + } + assertEquals(index.toString(), build, index.get("build").getAsString()); + // start() is not premain: it starts no JaCoCo and looks for none. + assertEquals(index.toString(), "unchecked", index.get("jacoco").getAsString()); + assertEquals(index.toString(), "", index.get("includes").getAsString()); + } + + private static String sha256(String text) { + try { + StringBuilder hex = new StringBuilder(); + for (byte b : MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.US_ASCII))) { + hex.append(String.format("%02x", b)); + } + return hex.toString(); + } catch (java.security.NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } + private void assertLogged(String prefix) { for (String line : logged) { if (line.startsWith(prefix)) { diff --git a/keploy-sdk/src/test/java/io/keploy/dedup/ControlSocketTest.java b/keploy-sdk/src/test/java/io/keploy/dedup/ControlSocketTest.java index 9ebdedb3..c365d74b 100644 --- a/keploy-sdk/src/test/java/io/keploy/dedup/ControlSocketTest.java +++ b/keploy-sdk/src/test/java/io/keploy/dedup/ControlSocketTest.java @@ -2,6 +2,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -16,6 +17,7 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.attribute.FileTime; import java.nio.file.attribute.PosixFilePermission; import java.util.List; import java.util.concurrent.BlockingQueue; @@ -78,6 +80,7 @@ public void close() { } }; private AFUNIXServerSocket foreign; + private static final String INDEX_START = "{\"kind\":\"index\","; @Before public void setUp() throws IOException { @@ -455,6 +458,72 @@ private static boolean controlThreadAlive() { return false; } + @Test + public void theIndexReportReachesTheReceiverOnceBeforeAnyCommand() throws Exception { + data = new DataReceiver(KeployDedupAgent.dataSocketPath); + startWithJacoco(); + String index = data.indexes.poll(30, TimeUnit.SECONDS); + assertNotNull("the INDEX report came with no command sent", index); + assertEquals(KeployDedupAgent.indexReport(), index); + assertEquals(TAKEN, send("START test-set-0/test-1")); + assertEquals(TAKEN, send("END test-set-0/test-1")); + assertTrue(data.next().contains("\"id\":\"test-set-0/test-1\"")); + assertEquals(TAKEN, send("START test-set-0/test-2")); + assertEquals(TAKEN, send("END test-set-0/test-2")); + assertTrue(data.next().contains("\"id\":\"test-set-0/test-2\"")); + assertNull("one INDEX report for one receiver", data.indexes.poll(200, TimeUnit.MILLISECONDS)); + } + + @Test + public void aReceiverBoundAnewGetsTheIndexReportBeforeItsFirstRecord() throws Exception { + startWithJacoco(); + // No receiver yet: the report sent once the socket was bound reached + // no one, and an ignored END publishes nothing, the report included. + assertEquals(IGNORED, send("END test-set-0/test-0")); + data = new DataReceiver(KeployDedupAgent.dataSocketPath); + assertNull(data.all.poll(200, TimeUnit.MILLISECONDS)); + assertEquals(TAKEN, send("START test-set-0/test-1")); + assertEquals(TAKEN, send("END test-set-0/test-1")); + assertEquals(KeployDedupAgent.indexReport(), data.all.poll(30, TimeUnit.SECONDS)); + assertTrue(data.all.poll(30, TimeUnit.SECONDS).contains("\"id\":\"test-set-0/test-1\"")); + + // The next test set's receiver (docker compose), on the same path. + data.close(); + // Keploy unlinks the old receiver's file before it binds the next. + new File(KeployDedupAgent.dataSocketPath).delete(); + data = new DataReceiver(KeployDedupAgent.dataSocketPath); + assertEquals(TAKEN, send("START test-set-1/test-1")); + assertEquals(TAKEN, send("END test-set-1/test-1")); + assertEquals(KeployDedupAgent.indexReport(), data.all.poll(30, TimeUnit.SECONDS)); + assertTrue(data.all.poll(30, TimeUnit.SECONDS).contains("\"id\":\"test-set-1/test-1\"")); + assertEquals(TAKEN, send("START test-set-1/test-2")); + assertEquals(TAKEN, send("END test-set-1/test-2")); + assertTrue(data.all.poll(30, TimeUnit.SECONDS).contains("\"id\":\"test-set-1/test-2\"")); + assertNull(data.all.poll(200, TimeUnit.MILLISECONDS)); + } + + @Test + public void aReceiverIsToldApartByItsFileAndTheTimeItWasMade() throws Exception { + // A file system can give a new socket file the inode of one just + // removed: the time it was made tells the two receivers apart. + File file = new File(dir, "receiver"); + Files.write(file.toPath(), new byte[] {1}); + Files.setLastModifiedTime(file.toPath(), FileTime.fromMillis(1_000_000_000_000L)); + Object key = KeployDedupAgent.receiverKey(file); + assertNotNull(key); + assertEquals(key, KeployDedupAgent.receiverKey(file)); + Files.setLastModifiedTime(file.toPath(), FileTime.fromMillis(1_000_000_001_000L)); + assertNotEquals(key, KeployDedupAgent.receiverKey(file)); + // Two receivers bound within one tick of the clock share a time: the + // file's identity tells them apart. + File other = new File(dir, "other"); + Files.write(other.toPath(), new byte[] {1}); + Files.setLastModifiedTime(other.toPath(), FileTime.fromMillis(1_000_000_001_000L)); + assertNotEquals(KeployDedupAgent.receiverKey(file), KeployDedupAgent.receiverKey(other)); + assertTrue(file.delete()); + assertNull("no file, no receiver", KeployDedupAgent.receiverKey(file)); + } + private void startWithJacoco() throws IOException { jacoco = new FakeJacoco(); System.setProperty(JACOCO_PORT_PROPERTY, String.valueOf(jacoco.server.getLocalPort())); @@ -509,11 +578,17 @@ private static String readAll(InputStream in) throws IOException { return new String(all.toByteArray(), StandardCharsets.UTF_8); } - /** Keploy's data socket: it keeps each payload sent to it. */ + /** + * Keploy's data socket: it keeps each payload sent to it, the tests' + * coverage records (next, poll) apart from the INDEX reports (indexes), + * and all of them in the order they came (all). + */ private static final class DataReceiver implements Closeable { final AFUNIXServerSocket server; final BlockingQueue payloads = new LinkedBlockingQueue<>(); + final BlockingQueue indexes = new LinkedBlockingQueue<>(); + final BlockingQueue all = new LinkedBlockingQueue<>(); DataReceiver(String path) throws IOException { server = AFUNIXServerSocket.newInstance(); @@ -527,7 +602,9 @@ private void serve() { while (!server.isClosed()) { try (Socket socket = server.accept()) { socket.setSoTimeout(30000); - payloads.add(readAll(socket.getInputStream())); + String payload = readAll(socket.getInputStream()); + all.add(payload); + (payload.startsWith(INDEX_START) ? indexes : payloads).add(payload); } catch (IOException e) { // Closed. } diff --git a/keploy-sdk/src/test/java/io/keploy/dedup/EmbeddedJacocoTest.java b/keploy-sdk/src/test/java/io/keploy/dedup/EmbeddedJacocoTest.java index 92cb3d1f..adf30542 100644 --- a/keploy-sdk/src/test/java/io/keploy/dedup/EmbeddedJacocoTest.java +++ b/keploy-sdk/src/test/java/io/keploy/dedup/EmbeddedJacocoTest.java @@ -108,7 +108,12 @@ public void aJacocoAgentOnTheClasspathOrAmongTheArgumentsCountsAsAttached() thro @Test public void theSdkAloneStartsJacocoOnTheAppsClassesOnly() throws Exception { - String output = runApp(new Run(SDK)); + Run run = new Run(SDK); + run.env.put("EMBEDDED_APP_INDEX", "1"); + String output = runApp(run); + // The INDEX report says what measures the app, and with what includes=. + assertTrue(output, output.contains("\"source\":\"override\"")); + assertTrue(output, output.contains("\"classes\":1,\"includes\":\"org.example.embedded.*\",\"jacoco\":\"embedded\"}")); assertTrue(output, output.contains("agent=true")); assertTrue(output, output.contains("app=true")); // Loaded after JaCoCo started, outside its includes=. @@ -124,6 +129,14 @@ public void aJacocoAgentAttachedAfterTheSdkIsLeftToRunAlone() throws Exception { assertLeftAlone(runApp(new Run(SDK, JACOCO))); } + @Test + public void theIndexReportNamesAJacocoAgentTheCommandAttached() throws Exception { + Run run = new Run(SDK, JACOCO); + run.env.put("EMBEDDED_APP_INDEX", "1"); + String output = runApp(run); + assertTrue(output, output.contains("\"classes\":1,\"includes\":\"\",\"jacoco\":\"attached\"}")); + } + @Test public void aJacocoAgentAfterTheSdkInJavaToolOptionsIsLeftToRunAlone() throws Exception { Run run = new Run(SDK, JACOCO); @@ -158,11 +171,14 @@ public void aJacocoThatCannotStartLeavesTheAppRunningUnmeasured() throws Excepti // measured. Run run = new Run(SDK); run.tmpdir = tmp.newFile("not-a-dir"); + run.env.put("EMBEDDED_APP_INDEX", "1"); String output = runApp(run); assertTrue(output, output.contains("agent=false")); assertTrue(output, output.contains("app=false")); assertTrue(output, output.contains("could not start the JaCoCo agent keploy-sdk.jar carries")); assertFalse(output, output.contains("started JaCoCo")); + // The INDEX report says nothing measures the app. + assertTrue(output, output.contains("\"includes\":\"\",\"jacoco\":\"none\"}")); } private static void assertLeftAlone(String output) { diff --git a/keploy-sdk/src/test/java/io/keploy/dedup/IndexReportTest.java b/keploy-sdk/src/test/java/io/keploy/dedup/IndexReportTest.java new file mode 100644 index 00000000..51ec8f72 --- /dev/null +++ b/keploy-sdk/src/test/java/io/keploy/dedup/IndexReportTest.java @@ -0,0 +1,95 @@ +package io.keploy.dedup; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; + +public class IndexReportTest { + + // Keploy's codecoverage.JavaBuildID of the ids 1, 0xff and + // 0xffffffffffffffff (its TestBuildIDs): sha256 over + // "0000000000000001\n00000000000000ff\nffffffffffffffff". + private static final String GO_BUILD_ID = "jv1-73ab094cbaa75db7d27e1cc885ca2260"; + + @Test + public void theBuildIdIsKeploysForTheSameIdsInAnyOrderAndRepeated() { + assertEquals(GO_BUILD_ID, IndexReport.buildId(new long[] {1, 0xff, 0xffffffffffffffffL})); + // A signed sort puts 0xffffffffffffffff (-1) first, a different id. + assertEquals(GO_BUILD_ID, IndexReport.buildId(new long[] {0xffffffffffffffffL, 1, 0xff, 1})); + assertEquals(GO_BUILD_ID, IndexReport.buildId(new long[] {0xff, 0xffffffffffffffffL, 0xff, 1})); + } + + @Test + public void noClassHasNoBuildId() { + assertEquals("", IndexReport.buildId(new long[0])); + } + + @Test + public void theClassIdIsJacocos() { + byte[] bytes = {(byte) 0xca, (byte) 0xfe, (byte) 0xba, (byte) 0xbe, 0, 0, 0, 52, 1, 2, 3}; + assertEquals(org.jacoco.core.internal.data.CRC64.classId(bytes), IndexReport.classId(bytes)); + } + + @Test + public void aRootIsTheLocationWithoutTheClassPath() { + assertEquals("/app/classes", IndexReport.rootOf("/app/classes/org/acme/Foo.class", "org/acme/Foo")); + assertEquals("/app/libs/x.jar", IndexReport.rootOf("/app/libs/x.jar!org/acme/Foo.class", "org/acme/Foo")); + assertEquals("/app/app.jar!BOOT-INF/classes", + IndexReport.rootOf("/app/app.jar!BOOT-INF/classes/org/acme/Foo.class", "org/acme/Foo")); + assertEquals("/app/app.jar!BOOT-INF/lib/x.jar", + IndexReport.rootOf("/app/app.jar!BOOT-INF/lib/x.jar!org/acme/Foo.class", "org/acme/Foo")); + // Not the class's own path: a name that only ends the same way. + assertNull(IndexReport.rootOf("/app/classes/org/xacme/Foo.class", "acme/Foo")); + assertNull(IndexReport.rootOf("/app/classes/org/acme/Bar.class", "org/acme/Foo")); + assertNull(IndexReport.rootOf("org/acme/Foo.class", "org/acme/Foo")); + // A class dir that is the file system's root. + assertEquals("/", IndexReport.rootOf("/org/acme/Foo.class", "org/acme/Foo")); + } + + @Test + public void theReportIsOneSmallJsonObjectWithNoClassList() { + IndexReport report = new IndexReport(GO_BUILD_ID, IndexReport.SOURCE_DERIVED, "java -jar /app/app.jar", + Arrays.asList("/app/libs/b.jar", "/app/classes", "/app/libs/b.jar"), + Collections.singletonList("org.acme"), 3, "org.acme.*", IndexReport.JACOCO_EMBEDDED); + JsonObject json = JsonParser.parseString(report.toJson()).getAsJsonObject(); + assertEquals("index", json.get("kind").getAsString()); + assertEquals(1, json.get("v").getAsInt()); + assertEquals(GO_BUILD_ID, json.get("build").getAsString()); + assertEquals("derived", json.get("source").getAsString()); + assertEquals("java -jar /app/app.jar", json.get("launch").getAsString()); + assertEquals("[\"/app/classes\",\"/app/libs/b.jar\"]", json.get("roots").toString()); + assertEquals(2, json.get("rootCount").getAsInt()); + assertEquals("[\"org.acme\"]", json.get("packages").toString()); + assertEquals(3, json.get("classes").getAsInt()); + assertEquals("org.acme.*", json.get("includes").getAsString()); + assertEquals("embedded", json.get("jacoco").getAsString()); + assertEquals(11, json.size()); + // Its kind starts the object, so a reader can tell it from a test's + // coverage record by its first bytes. + assertEquals(0, report.toJson().indexOf("{\"kind\":\"index\",")); + // A record of no test: old Keploy reads no test id from it. + assertFalse(json.has("id")); + } + + @Test + public void manyRootsAreCutAndCounted() { + List roots = new ArrayList<>(); + for (int i = 0; i < IndexReport.MAX_ROOTS + 6; i++) { + roots.add(String.format("/cp/lib-%03d.jar", i)); + } + IndexReport report = new IndexReport("", IndexReport.SOURCE_CLASSPATH, "why", roots, + Collections.emptyList(), 0, "", IndexReport.JACOCO_UNCHECKED); + assertEquals(IndexReport.MAX_ROOTS, report.roots.size()); + assertEquals(IndexReport.MAX_ROOTS + 6, report.rootCount); + assertEquals("/cp/lib-000.jar", report.roots.get(0)); + } +} diff --git a/keploy-sdk/src/test/java/org/example/embedded/EmbeddedApp.java b/keploy-sdk/src/test/java/org/example/embedded/EmbeddedApp.java index 8f6e5a01..d4d7e665 100644 --- a/keploy-sdk/src/test/java/org/example/embedded/EmbeddedApp.java +++ b/keploy-sdk/src/test/java/org/example/embedded/EmbeddedApp.java @@ -6,7 +6,7 @@ * EmbeddedApp is the app of EmbeddedJacocoTest's own JVMs: it says whether a * JaCoCo agent runs (agent=false when none was attached or started), and * whether JaCoCo instrumented its own class and a library class it loads only - * now, after every agent's premain. + * now, after every agent's premain, and the INDEX report the SDK sends. */ public final class EmbeddedApp { @@ -24,6 +24,18 @@ public static void main(String[] args) throws Exception { System.out.println("agent=" + (agent != null)); System.out.println("app=" + instrumented(EmbeddedApp.class)); System.out.println("library=" + instrumented(Class.forName("org.junit.runner.Description"))); + if (System.getenv("EMBEDDED_APP_INDEX") != null) { + // Reads the index when premain did not. + System.out.println("index=" + indexReport()); + } + } + + // indexReport is the INDEX report the SDK sends Keploy, or null when no + // SDK started. + private static Object indexReport() throws Exception { + Method report = Class.forName("io.keploy.dedup.KeployDedupAgent").getDeclaredMethod("indexReport"); + report.setAccessible(true); + return report.invoke(null); } private static boolean instrumented(Class type) { diff --git a/scripts/smoke-javaagent.sh b/scripts/smoke-javaagent.sh index af75bdd3..27b6bdac 100755 --- a/scripts/smoke-javaagent.sh +++ b/scripts/smoke-javaagent.sh @@ -73,6 +73,10 @@ public final class SmokeHarness { public static void main(String[] args) throws Exception { String mode = args.length == 0 ? "unknown" : args[0]; + // What the INDEX report must say: where the app's classes came from, + // and which JaCoCo measures them. + String wantSource = args.length > 1 ? args[1] : ""; + String wantJacoco = args.length > 2 ? args[2] : ""; // Keploy fires its first test as soon as the app answers a health // check, so the agent has to be listening before main runs. This START @@ -82,17 +86,29 @@ public final class SmokeHarness { delete(DATA_SOCKET); + // Keploy's receiver: one record per connection. This one is bound + // after the START: no socket file was there when the SDK bound its + // control socket, so it sent no INDEX report then, and it must send + // it here first, before the test's coverage. + AtomicReference index = new AtomicReference(); AtomicReference payload = new AtomicReference(); CountDownLatch received = new CountDownLatch(1); Thread receiver = new Thread(new Runnable() { @Override public void run() { try (AFUNIXServerSocket server = AFUNIXServerSocket.newInstance()) { - server.bind(AFUNIXSocketAddress.of(DATA_SOCKET), 1); - try (AFUNIXSocket socket = server.accept()) { - payload.set(readAll(socket.getInputStream())); - received.countDown(); + server.bind(AFUNIXSocketAddress.of(DATA_SOCKET), 8); + for (int n = 0; payload.get() == null; n++) { + try (AFUNIXSocket socket = server.accept()) { + String record = readAll(socket.getInputStream()); + if (n == 0 && record.startsWith("{\"kind\":\"index\",")) { + index.set(record); + } else { + payload.set(record); + } + } } + received.countDown(); } catch (Exception e) { e.printStackTrace(); } @@ -120,6 +136,15 @@ public final class SmokeHarness { || !json.contains("smoke/Work")) { throw new IllegalStateException("unexpected coverage payload for " + mode + ": " + json); } + String report = index.get(); + if (report == null + || !report.contains("\"source\":\"" + wantSource + "\"") + || !report.contains("\"jacoco\":\"" + wantJacoco + "\"") + || ("embedded".equals(wantJacoco) != report.matches(".*\"includes\":\"[^\"]*smoke\\.Work.*"))) { + throw new IllegalStateException("the first record on the data socket was no INDEX report with source " + + wantSource + " and jacoco " + wantJacoco + " in " + mode + ": " + report); + } + System.out.println(mode + ": " + report); System.out.println(mode + ": " + json); } @@ -213,7 +238,7 @@ cp -R "${work_dir}/classes/." "${work_dir}/gradle/build/classes/java/main/" run_smoke "maven-classes" \ java_with_agents "maven-classes" \ -cp "target/classes:${agent_jar}" \ - smoke.SmokeHarness "maven-classes" + smoke.SmokeHarness "maven-classes" derived attached ) ( @@ -221,13 +246,16 @@ cp -R "${work_dir}/classes/." "${work_dir}/gradle/build/classes/java/main/" run_smoke "gradle-classes" \ java_with_agents "gradle-classes" \ -cp "build/classes/java/main:${agent_jar}" \ - smoke.SmokeHarness "gradle-classes" + smoke.SmokeHarness "gradle-classes" derived attached ) +# Named for the whole-classpath fallback this layout once took: the agent now +# takes the dir that holds the main class (its package, smoke, has one part, +# so the INDEX lists no package root). run_smoke "classpath-fallback" \ java_with_agents "classpath-fallback" \ -cp "${work_dir}/classes:${agent_jar}" \ - smoke.SmokeHarness "classpath-fallback" + smoke.SmokeHarness "classpath-fallback" derived attached mkdir -p "${work_dir}/jar/lib" cp "${agent_jar}" "${work_dir}/jar/lib/keploy-sdk.jar" @@ -242,7 +270,7 @@ jar cfm "${work_dir}/jar/plain-app.jar" "${work_dir}/jar/MANIFEST.MF" -C "${work cd "${work_dir}/jar" run_smoke "plain-jar" \ java_with_agents "plain-jar" \ - -jar plain-app.jar "plain-jar" + -jar plain-app.jar "plain-jar" derived attached ) # keploy-sdk.jar alone: it starts the JaCoCo agent it carries, with includes= @@ -264,14 +292,14 @@ java_with_sdk_only() { run_smoke "sdk-only-maven-classes" \ java_with_sdk_only "sdk-only-maven-classes" \ -cp "target/classes:${agent_jar}" \ - smoke.SmokeHarness "sdk-only-maven-classes" + smoke.SmokeHarness "sdk-only-maven-classes" derived embedded ) ( cd "${work_dir}/jar" run_smoke "sdk-only-plain-jar" \ java_with_sdk_only "sdk-only-plain-jar" \ - -jar plain-app.jar "sdk-only-plain-jar" + -jar plain-app.jar "sdk-only-plain-jar" derived embedded ) mkdir -p "${work_dir}/boot-root/BOOT-INF/classes" "${work_dir}/war-root/WEB-INF/classes" @@ -284,13 +312,13 @@ KEPLOY_JAVA_CLASS_DIRS="${work_dir}/spring-boot-layout.jar" \ run_smoke "spring-boot-layout" \ java_with_agents "spring-boot-layout" \ -cp "${work_dir}/classes:${agent_jar}" \ - smoke.SmokeHarness "spring-boot-layout" + smoke.SmokeHarness "spring-boot-layout" override attached KEPLOY_JAVA_CLASS_DIRS="${work_dir}/servlet-layout.war" \ run_smoke "servlet-war-layout" \ java_with_agents "servlet-war-layout" \ -cp "${work_dir}/classes:${agent_jar}" \ - smoke.SmokeHarness "servlet-war-layout" + smoke.SmokeHarness "servlet-war-layout" override attached # The agent binds the control socket in premain, so junixsocket's native # library loads there, before main runs. A JVM that cannot load it must still From 9a7b1f91ce4126378e1db29facc1558c54ff4440 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Fri, 20 Nov 2026 00:02:43 +0530 Subject: [PATCH 4/5] fix(dedup): carry ASM, Gson and JaCoCo core renamed in keploy-sdk.jar keploy-sdk.jar is a -javaagent jar, so the JVM puts it on the app's system class path. It carried JaCoCo's core, the ASM that core needs, and Gson under their own package names, so they met the app's own copies: - Spring Boot's launcher loads BOOT-INF/lib through a loader that asks the system class loader first. An app that ships its own ASM, Gson or JaCoCo core got keploy-sdk's copy instead, in keploy-sdk's version: a NoSuchMethodError or LinkageError in the user's app once it calls an API that version lacks. - A launch that puts the app's jars first (Jib's -cp /app/libs/*) gave keploy-sdk the app's version instead. With an older JaCoCo core there, keploy-sdk sent no INDEX record at all, and said nothing about it. The shade plugin now relocates org.jacoco.core, org.objectweb.asm and com.google.gson under io.keploy.shaded. junixsocket keeps its names: its native library is bound to them. The jacocoagent.jar that keploy-sdk starts already carries its own renamed runtime. The shade plugin also rewrites every string constant that starts with a relocated package, and the list of keploy-sdk's own classes, which are never fingerprinted as the app's, named org/objectweb/asm/ and com/google/gson/. Left as they were, the jar would have fingerprinted its own renamed JaCoCo core. The list now names keploy-sdk.jar's classes as the jar holds them: io/keploy/shaded/, the JaCoCo agent under org/jacoco/, and all of junixsocket under org/newsclub/. That last one fixes an older gap: junixsocket's org/newsclub/lib class was not on the list, so an override that named keploy-sdk.jar fingerprinted it and passed it to JaCoCo's includes=. An app's own ASM or Gson is now one of the app's libraries, like Jackson, and is fingerprinted when the index takes in libraries (the whole jar or the whole class path). THIRD-PARTY-NOTICES.txt says which packages are renamed and to what. Tests: - AppIndexTest: an override that takes in keploy-sdk.jar's classes (its renamed libraries, the JaCoCo agent, both junixsocket packages) indexes the app's own classes and its own ASM and Gson only. - smoke-javaagent.sh, with keploy-sdk.jar alone, against the app's own ASM 5.2, Gson 2.11.0 and JaCoCo core 0.7.9: - those jars on the class path before keploy-sdk.jar: the INDEX record and the test's coverage still arrive; - those jars in a loader under the system class loader, as Spring Boot loads BOOT-INF/lib: the app gets its own ASM, Gson and JaCoCo core; - KEPLOY_JAVA_CLASS_DIRS that names keploy-sdk.jar too: the INDEX record counts the app's classes alone. On the jar from before this change, all three cases fail. Signed-off-by: slayerjain --- keploy-sdk/pom.xml | 23 ++++++ .../io/keploy/dedup/KeployDedupAgent.java | 13 ++- .../META-INF/THIRD-PARTY-NOTICES.txt | 12 ++- .../java/io/keploy/dedup/AppIndexTest.java | 27 +++++++ scripts/smoke-javaagent.sh | 79 ++++++++++++++++++- 5 files changed, 146 insertions(+), 8 deletions(-) diff --git a/keploy-sdk/pom.xml b/keploy-sdk/pom.xml index 1e1d3714..9625560b 100644 --- a/keploy-sdk/pom.xml +++ b/keploy-sdk/pom.xml @@ -124,6 +124,29 @@ + + + + org.jacoco.core + io.keploy.shaded.org.jacoco.core + + + org.objectweb.asm + io.keploy.shaded.org.objectweb.asm + + + com.google.gson + io.keploy.shaded.com.google.gson + + *:* diff --git a/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java b/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java index 26a442a5..e273bdbf 100644 --- a/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java +++ b/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java @@ -1816,15 +1816,22 @@ private String classKeyFromJarEntry(String entryName) { return entryName; } + // shouldSkipClass keeps keploy-sdk.jar's own classes out of the index: + // its code, the ASM, Gson and JaCoCo core it carries renamed under + // io/keploy/shaded, JaCoCo's agent (the AgentJar it starts, and the + // runtime of any JaCoCo agent), and junixsocket, which it carries under + // its own names because its native library is bound to them. An app's + // own ASM or Gson is one of the app's libraries, as Jackson is. The + // shade plugin rewrites every string constant that starts with a + // package it renames, so no prefix here may start with one. private boolean shouldSkipClass(String name) { return name.endsWith("module-info.class") || name.endsWith("package-info.class") || name.startsWith("io/keploy/dedup/") || name.startsWith("io/keploy/servlet/") + || name.startsWith("io/keploy/shaded/") || name.startsWith("org/jacoco/") - || name.startsWith("org/objectweb/asm/") - || name.startsWith("org/newsclub/net/unix/") - || name.startsWith("com/google/gson/") + || name.startsWith("org/newsclub/") || name.contains("$Mockito") || name.contains("Test.class"); } diff --git a/keploy-sdk/src/main/resources/META-INF/THIRD-PARTY-NOTICES.txt b/keploy-sdk/src/main/resources/META-INF/THIRD-PARTY-NOTICES.txt index 442d0e08..028b7242 100644 --- a/keploy-sdk/src/main/resources/META-INF/THIRD-PARTY-NOTICES.txt +++ b/keploy-sdk/src/main/resources/META-INF/THIRD-PARTY-NOTICES.txt @@ -6,12 +6,16 @@ JaCoCo 0.8.15 - Eclipse Public License 2.0 (https://www.eclipse.org/legal/epl-2. https://github.com/jacoco/jacoco, tag v0.8.15. - jacocoagent.jar, unchanged, from org.jacoco:org.jacoco.agent:0.8.15; keploy-sdk starts it when the JVM attaches no JaCoCo agent of its own. - - the classes of org.jacoco:org.jacoco.core:0.8.15, unchanged. + - the classes of org.jacoco:org.jacoco.core:0.8.15, unchanged but for their + package, renamed from org.jacoco.core to io.keploy.shaded.org.jacoco.core + (maven-shade-plugin relocation, keploy-sdk/pom.xml), so that they never meet + an app's own JaCoCo. ASM 9.10.1 - BSD 3-Clause License Source: https://gitlab.ow2.org/asm/asm - - the classes of org.ow2.asm:asm, asm-commons and asm-tree 9.10.1, unchanged - (under org/objectweb/asm), which org.jacoco.core depends on, and the copy + - the classes of org.ow2.asm:asm, asm-commons and asm-tree 9.10.1, which + org.jacoco.core depends on, unchanged but for their package, renamed from + org.objectweb.asm to io.keploy.shaded.org.objectweb.asm; and the copy inside jacocoagent.jar. ASM is subject to the following license: ASM: a very small and fast Java bytecode manipulation framework @@ -44,6 +48,8 @@ THE POSSIBILITY OF SUCH DAMAGE. Gson 2.9.1 - Apache License 2.0 Source: https://github.com/google/gson + - its classes, unchanged but for their package, renamed from com.google.gson + to io.keploy.shaded.com.google.gson. junixsocket 2.10.1 - Apache License 2.0 Source: https://github.com/kohlschutter/junixsocket diff --git a/keploy-sdk/src/test/java/io/keploy/dedup/AppIndexTest.java b/keploy-sdk/src/test/java/io/keploy/dedup/AppIndexTest.java index 0c3be885..7c568725 100644 --- a/keploy-sdk/src/test/java/io/keploy/dedup/AppIndexTest.java +++ b/keploy-sdk/src/test/java/io/keploy/dedup/AppIndexTest.java @@ -141,6 +141,33 @@ public void configuredClassDirsAreFingerprintedAndWarmedUp() throws Exception { assertNotLogged("found no class in KEPLOY_JAVA_CLASS_DIRS"); } + @Test + public void keploySdksOwnClassesAreNeverTheApps() throws Exception { + // An override that takes in keploy-sdk.jar's classes too: its code, + // the libraries it carries renamed under io/keploy/shaded, JaCoCo's + // agent and junixsocket stay out. The app's own ASM and Gson are the + // app's libraries, so they are fingerprinted, as Jackson would be. + File jar = jar(tmp.newFile("app.jar"), + "com/acme/shop/Main.class", + "org/objectweb/asm/ClassReader.class", + "com/google/gson/Gson.class", + "io/keploy/dedup/KeployDedupAgent.class", + "io/keploy/servlet/KeployMiddleware.class", + "io/keploy/shaded/org/objectweb/asm/ClassReader.class", + "io/keploy/shaded/com/google/gson/Gson.class", + "io/keploy/shaded/org/jacoco/core/JaCoCo.class", + "org/jacoco/agent/AgentJar.class", + "org/newsclub/net/unix/AFUNIXSocket.class", + "org/newsclub/lib/junixsocket/common/NarMetadata.class"); + System.setProperty("keploy.java.class.dirs", jar.getPath()); + System.setProperty("java.class.path", jar.getPath()); + System.setProperty("sun.java.command", "com.acme.shop.Main"); + + startAndSendStart(); + + assertIndex("override", "the class dirs [" + jar.getPath() + "]", "[]", 3, jar.getPath()); + } + @Test public void manyClassDirsAreSaidWithHowManyMore() throws Exception { List dirs = new ArrayList<>(); diff --git a/scripts/smoke-javaagent.sh b/scripts/smoke-javaagent.sh index 27b6bdac..9716be29 100755 --- a/scripts/smoke-javaagent.sh +++ b/scripts/smoke-javaagent.sh @@ -62,6 +62,8 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.InputStream; import java.io.OutputStream; +import java.net.URL; +import java.net.URLClassLoader; import java.nio.charset.StandardCharsets; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -77,6 +79,8 @@ public final class SmokeHarness { // and which JaCoCo measures them. String wantSource = args.length > 1 ? args[1] : ""; String wantJacoco = args.length > 2 ? args[2] : ""; + // How many classes the index must hold, when the case says. + String wantClasses = args.length > 3 ? args[3] : ""; // Keploy fires its first test as soon as the app answers a health // check, so the agent has to be listening before main runs. This START @@ -140,14 +144,52 @@ public final class SmokeHarness { if (report == null || !report.contains("\"source\":\"" + wantSource + "\"") || !report.contains("\"jacoco\":\"" + wantJacoco + "\"") - || ("embedded".equals(wantJacoco) != report.matches(".*\"includes\":\"[^\"]*smoke\\.Work.*"))) { + || ("embedded".equals(wantJacoco) != report.matches(".*\"includes\":\"[^\"]*smoke\\.Work.*")) + || (!wantClasses.isEmpty() && !report.matches(".*\"classes\":" + wantClasses + "[,}].*"))) { throw new IllegalStateException("the first record on the data socket was no INDEX report with source " - + wantSource + " and jacoco " + wantJacoco + " in " + mode + ": " + report); + + wantSource + ", jacoco " + wantJacoco + + (wantClasses.isEmpty() ? "" : " and " + wantClasses + " classes") + " in " + mode + ": " + report); + } + String ownLibraries = System.getProperty("smoke.ownLibraries", ""); + if (!ownLibraries.isEmpty()) { + loadsItsOwn(ownLibraries); } System.out.println(mode + ": " + report); System.out.println(mode + ": " + json); } + // loadsItsOwn loads ASM, Gson and JaCoCo's core from the app's own jars + // as Spring Boot's launcher loads BOOT-INF/lib: through a loader that asks + // its parent, the system class loader, first. keploy-sdk.jar is on the + // system class path, so a copy of these it carried under their own names + // would be what the app gets, in keploy-sdk's version, not the app's. + private static void loadsItsOwn(String jars) throws Exception { + String[] paths = jars.split(File.pathSeparator); + URL[] urls = new URL[paths.length]; + for (int i = 0; i < paths.length; i++) { + urls[i] = new File(paths[i]).toURI().toURL(); + } + URLClassLoader libraries = new URLClassLoader(urls, ClassLoader.getSystemClassLoader()); + try { + for (String name : new String[] { + "org.objectweb.asm.ClassReader", "com.google.gson.Gson", "org.jacoco.core.JaCoCo"}) { + Class type = Class.forName(name, false, libraries); + URL from = type.getProtectionDomain().getCodeSource() == null + ? null : type.getProtectionDomain().getCodeSource().getLocation(); + boolean own = false; + for (URL url : urls) { + own |= from != null && new File(from.toURI()).equals(new File(url.toURI())); + } + if (!own) { + throw new IllegalStateException("the app's " + name + " came from " + from + + ", not from the app's own jars " + jars); + } + } + } finally { + libraries.close(); + } + } + private static void command(String command) throws Exception { try (AFUNIXSocket socket = AFUNIXSocket.newInstance()) { socket.connect(AFUNIXSocketAddress.of(CONTROL_SOCKET), 3000); @@ -302,6 +344,39 @@ java_with_sdk_only() { -jar plain-app.jar "sdk-only-plain-jar" derived embedded ) +# The app's own ASM, Gson and JaCoCo core, in versions other than the ones +# keploy-sdk.jar carries. keploy-sdk.jar carries its copies renamed under +# io/keploy/shaded, so the app gets its own and keploy-sdk its own, whichever +# comes first: the app's on the class path before keploy-sdk.jar (Jib's +# launch), or the app's in a loader under the system class loader (Spring +# Boot's BOOT-INF/lib). +for artifact in org.ow2.asm:asm:5.2 com.google.code.gson:gson:2.11.0 org.jacoco:org.jacoco.core:0.7.9; do + mvn -q -f "${repo_root}/pom.xml" org.apache.maven.plugins:maven-dependency-plugin:3.6.1:copy \ + -Dartifact="${artifact}" -DoutputDirectory="${work_dir}/app-libs" +done +app_libs="$(find "${work_dir}/app-libs" -name '*.jar' | sort | paste -sd: -)" + +run_smoke "sdk-only-app-libraries-first" \ + java_with_sdk_only "sdk-only-app-libraries-first" \ + -cp "${app_libs}:${work_dir}/classes:${agent_jar}" \ + smoke.SmokeHarness "sdk-only-app-libraries-first" derived embedded + +run_smoke "sdk-only-app-libraries-in-own-loader" \ + java_with_sdk_only "sdk-only-app-libraries-in-own-loader" \ + -Dsmoke.ownLibraries="${app_libs}" \ + -cp "${work_dir}/classes:${agent_jar}" \ + smoke.SmokeHarness "sdk-only-app-libraries-in-own-loader" derived embedded + +# keploy-sdk.jar's own classes, the libraries it carries among them, are +# never the app's: an override that names keploy-sdk.jar still indexes the +# app's classes alone. +app_class_count="$(find "${work_dir}/classes" -name '*.class' | wc -l | tr -d ' ')" +KEPLOY_JAVA_CLASS_DIRS="${work_dir}/classes:${agent_jar}" \ +run_smoke "sdk-only-override-names-keploy-sdk" \ + java_with_sdk_only "sdk-only-override-names-keploy-sdk" \ + -cp "${work_dir}/classes:${agent_jar}" \ + smoke.SmokeHarness "sdk-only-override-names-keploy-sdk" override embedded "${app_class_count}" + mkdir -p "${work_dir}/boot-root/BOOT-INF/classes" "${work_dir}/war-root/WEB-INF/classes" cp -R "${work_dir}/classes/." "${work_dir}/boot-root/BOOT-INF/classes/" cp -R "${work_dir}/classes/." "${work_dir}/war-root/WEB-INF/classes/" From 6d710cecd190cc233265c90bf97b63f854b7354e Mon Sep 17 00:00:00 2001 From: slayerjain Date: Fri, 20 Nov 2026 04:15:18 +0530 Subject: [PATCH 5/5] feat(dedup): name keploy-sdk's and JaCoCo's versions in the INDEX record Only the JVM knows which keploy-sdk.jar it runs and which JaCoCo agent measures the app's classes, and Keploy needs both to say what produced a test's coverage. The INDEX record now carries them, after its other fields: - sdkVersion: the Implementation-Version of keploy-sdk.jar's manifest, which the build sets to the project's version. It is read only when that manifest's Implementation-Title is keploy-sdk's own, so an app jar that shades or copies the SDK's classes never lends its version. It is "" when not known: the SDK's classes come from a build's class dir, or from a jar that is not keploy-sdk.jar. - jacocoVersion: the version the measuring JaCoCo agent's jar names ("0.8.15"). That is the agent keploy-sdk.jar carries and started (jacoco embedded), or the one the command or JAVA_TOOL_OPTIONS attached (attached), read from the jar that holds its runtime, on the system class path or named by a -javaagent argument. It is "" when nothing measures the classes, the SDK was not started by premain, the jar names no version or cannot be read, or the runtime is in a class dir or in a jar nested in another. The record's v stays 1: both fields are new and come last, and a reader that does not know them skips them. EmbeddedJacoco.attached now returns the attached agent's version, or null when there is none, in place of a boolean. A jar that holds JaCoCo's runtime counts as attached whatever its manifest says, as before. Tests: IndexReportTest checks both fields and their place at the end of the record, and versionOf for a class loaded from keploy-sdk's jar with and without a version, from a jar with another title or none, and from a class dir. EmbeddedJacocoTest checks the version attached returns for an agent on the class path and among the arguments, "" for a JaCoCo runtime in a jar that names no version, in a class dir, or in a jar nested in another, and the versions in the INDEX of forked JVMs: the SDK alone, JaCoCo attached after it and before it, and a JaCoCo that could not start. scripts/smoke-javaagent.sh checks, for every launch that sends an INDEX, that it names keploy-sdk.jar's version and the version of the JaCoCo agent that measures the app (the 0.8.12 agent the script attaches, or the one keploy-sdk.jar carries), each as its jar's manifest names it. Signed-off-by: slayerjain --- README.md | 2 +- .../java/io/keploy/dedup/EmbeddedJacoco.java | 72 ++++++++++++--- .../java/io/keploy/dedup/IndexReport.java | 34 ++++++- .../io/keploy/dedup/KeployDedupAgent.java | 15 ++-- .../java/io/keploy/dedup/AppIndexTest.java | 3 + .../io/keploy/dedup/EmbeddedJacocoTest.java | 89 ++++++++++++++++-- .../java/io/keploy/dedup/IndexReportTest.java | 90 ++++++++++++++++++- scripts/smoke-javaagent.sh | 36 +++++++- 8 files changed, 311 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 540f1097..12e4edb2 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Keploy Enterprise drives dynamic dedup per testcase. Coverage is collected at per-testcase granularity, not process granularity. -Before the first testcase's coverage, the agent sends Enterprise one INDEX record on `/tmp/coverage_data.sock`: a JSON object starting with `{"kind":"index",` that says how it found the application's own classes (`source`: `override` when the dirs and jars in `KEPLOY_JAVA_CLASS_DIRS` held them, `derived` from the launch or from `target/classes` or `build/classes/java/main` under the working directory, `whole-jar` or `classpath` when it could not tell them from the libraries, `none`), what it found them from in words (`launch`), the dirs and jars it read them from (`roots`, at most 64, and `rootCount`), the package roots it kept (`packages`), how many classes it indexed, the build id of those classes (`jv1-` and 32 hex), which JaCoCo agent measures them (`jacoco`: `embedded`, the one keploy-sdk.jar starts, `attached` by the command, `none`, or `unchecked` when the agent was not started as a `-javaagent`), and that embedded agent's `includes=`. It carries no class list. It is sent as the agent starts when that path's socket file is there (Enterprise's receiver makes it), and otherwise, or when that send fails, right before the first testcase's coverage; a receiver Enterprise binds anew on that path gets it again before its first record. On the TCP transport used in Kubernetes no INDEX record is sent. +Before the first testcase's coverage, the agent sends Enterprise one INDEX record on `/tmp/coverage_data.sock`: a JSON object starting with `{"kind":"index",` that says how it found the application's own classes (`source`: `override` when the dirs and jars in `KEPLOY_JAVA_CLASS_DIRS` held them, `derived` from the launch or from `target/classes` or `build/classes/java/main` under the working directory, `whole-jar` or `classpath` when it could not tell them from the libraries, `none`), what it found them from in words (`launch`), the dirs and jars it read them from (`roots`, at most 64, and `rootCount`), the package roots it kept (`packages`), how many classes it indexed, the build id of those classes (`jv1-` and 32 hex), which JaCoCo agent measures them (`jacoco`: `embedded`, the one keploy-sdk.jar starts, `attached` by the command, `none`, or `unchecked` when the agent was not started as a `-javaagent`), that embedded agent's `includes=`, and two versions as their jars' manifests name them, or `""` when not known: keploy-sdk.jar's (`sdkVersion`) and that of the JaCoCo agent that measures the classes, embedded or attached (`jacocoVersion`). It carries no class list. It is sent as the agent starts when that path's socket file is there (Enterprise's receiver makes it), and otherwise, or when that send fails, right before the first testcase's coverage; a receiver Enterprise binds anew on that path gets it again before its first record. On the TCP transport used in Kubernetes no INDEX record is sent. The agent binds `/tmp/coverage_control.sock` before the application's `main` runs, so the socket is listening before the application can answer a health check. It answers `ACK` to each command it took, and `IGNORED` to an `END` that no `START` for that testcase came before: nothing is published for it, and Enterprise keeps that testcase out of the comparison. On the TCP transport used in Kubernetes the same `END` is answered with `IGNORED ` and then `ACK`. diff --git a/keploy-sdk/src/main/java/io/keploy/dedup/EmbeddedJacoco.java b/keploy-sdk/src/main/java/io/keploy/dedup/EmbeddedJacoco.java index 24d7cee7..72121b69 100644 --- a/keploy-sdk/src/main/java/io/keploy/dedup/EmbeddedJacoco.java +++ b/keploy-sdk/src/main/java/io/keploy/dedup/EmbeddedJacoco.java @@ -6,6 +6,9 @@ import java.io.IOException; import java.io.InputStream; import java.lang.instrument.Instrumentation; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; import java.nio.file.Files; import java.util.Collection; import java.util.List; @@ -47,15 +50,19 @@ private EmbeddedJacoco() { } /** - * attached says whether the JVM has a JaCoCo agent of its own: one the - * command or JAVA_TOOL_OPTIONS attaches with -javaagent. An agent before + * attached finds the JVM's own JaCoCo agent: one the command or + * JAVA_TOOL_OPTIONS attaches with -javaagent. An agent before * keploy-sdk.jar is on the system classpath already; one after it is * not until its own premain, so it is found by its jar among the JVM's - * arguments (the RuntimeMXBean's, JAVA_TOOL_OPTIONS' among them). + * arguments (the RuntimeMXBean's, JAVA_TOOL_OPTIONS' among them). It is + * null when there is none, else the Implementation-Version its jar's + * manifest names: "" when it names none, or the agent is in no jar file + * this can read. */ - static boolean attached(ClassLoader system, List jvmArguments) { - if (system.getResource(RUNTIME_CLASS) != null) { - return true; + static String attached(ClassLoader system, List jvmArguments) { + URL runtime = system.getResource(RUNTIME_CLASS); + if (runtime != null) { + return versionOfJarHolding(runtime); } for (String argument : jvmArguments) { if (!argument.startsWith(JAVAAGENT)) { @@ -65,13 +72,55 @@ static boolean attached(ClassLoader system, List jvmArguments) { int options = path.indexOf('='); try (JarFile jar = new JarFile(options < 0 ? path : path.substring(0, options))) { if (jar.getEntry(RUNTIME_CLASS) != null) { - return true; + return versionOf(jar); } } catch (IOException e) { // The JVM cannot load an agent from it either. } } - return false; + return null; + } + + // versionOf is the Implementation-Version jar's manifest names, or "" + // when it names none or its manifest cannot be read: a jar that holds + // the runtime is an attached agent whatever its manifest says. + private static String versionOf(JarFile jar) { + try { + return implementationVersion(jar.getManifest()); + } catch (IOException e) { + return ""; + } + } + + // versionOfJarHolding is the Implementation-Version of the jar file + // holding the resource at url ("jar:file:/x/jacocoagent.jar!/org/..."), + // or "" when its manifest names none, url is in no jar file (a class + // dir), in a jar nested in another ("...app.jar!/lib/x.jar!/org/..."), + // or the jar cannot be read. + static String versionOfJarHolding(URL url) { + String spec = url.toString(); + int entry = spec.indexOf("!/"); + if (!"jar".equals(url.getProtocol()) || entry < 0 || spec.indexOf("!/", entry + 2) >= 0) { + return ""; + } + try { + URI jar = new URI(spec.substring("jar:".length(), entry)); + if (!"file".equalsIgnoreCase(jar.getScheme())) { + return ""; + } + try (JarFile file = new JarFile(new File(jar))) { + return versionOf(file); + } + } catch (IOException | URISyntaxException | IllegalArgumentException e) { + return ""; + } + } + + // implementationVersion is the Implementation-Version manifest names, + // or "" when it names none or there is no manifest. + static String implementationVersion(Manifest manifest) { + String version = manifest == null ? null : manifest.getMainAttributes().getValue("Implementation-Version"); + return version == null ? "" : version; } /** @@ -143,7 +192,8 @@ static String options(String includes) { * start writes the JaCoCo agent jar into a new temp dir only this user * can read, puts it on the system classpath as the JVM does a * -javaagent jar, and runs its premain with options. It returns - * JaCoCo's version. The jar is removed when the JVM exits, except on + * JaCoCo's version as the jar's manifest names it ("0.8.15"; "" when it + * names none). The jar is removed when the JVM exits, except on * Windows, where the JVM holds it open. */ static String start(Instrumentation instrumentation, String options) throws Exception { @@ -161,7 +211,7 @@ static String start(Instrumentation instrumentation, String options) throws Exce Manifest manifest = jarFile.getManifest(); Attributes attributes = manifest == null ? new Attributes() : manifest.getMainAttributes(); premain = attributes.getValue("Premain-Class"); - version = attributes.getValue("Implementation-Version"); + version = implementationVersion(manifest); if (premain == null) { throw new IllegalStateException("the JaCoCo agent jar names no Premain-Class"); } @@ -170,6 +220,6 @@ static String start(Instrumentation instrumentation, String options) throws Exce Class.forName(premain, true, ClassLoader.getSystemClassLoader()) .getMethod("premain", String.class, Instrumentation.class) .invoke(null, options, instrumentation); - return version == null ? "" : version; + return version; } } diff --git a/keploy-sdk/src/main/java/io/keploy/dedup/IndexReport.java b/keploy-sdk/src/main/java/io/keploy/dedup/IndexReport.java index 32530bf6..71ac0cd0 100644 --- a/keploy-sdk/src/main/java/io/keploy/dedup/IndexReport.java +++ b/keploy-sdk/src/main/java/io/keploy/dedup/IndexReport.java @@ -67,9 +67,17 @@ final class IndexReport { // includes= of the JaCoCo agent the SDK started, or "". final String includes; final String jacoco; + // keploy-sdk.jar's version (versionOf), or "" when it is not known: the + // SDK's classes come from a build's class dir, or from a jar that is not + // keploy-sdk.jar. + final String sdkVersion; + // The version of the JaCoCo agent that measures the classes: the one the + // SDK started or the one the command attached, as its jar names it; "" + // when none does (jacoco none or unchecked) or its jar names none. + final String jacocoVersion; IndexReport(String build, String source, String launch, Collection roots, List packages, - int classes, String includes, String jacoco) { + int classes, String includes, String jacoco, String sdkVersion, String jacocoVersion) { this.build = build; this.source = source; this.launch = launch; @@ -80,6 +88,30 @@ final class IndexReport { this.classes = classes; this.includes = includes; this.jacoco = jacoco; + this.sdkVersion = sdkVersion; + this.jacocoVersion = jacocoVersion; + } + + // SDK_TITLE is the Implementation-Title keploy-sdk.jar's build writes + // in its manifest, next to its version: the project's name. + static final String SDK_TITLE = "Keploy Java Coverage Agent"; + + /** + * versionOf is the Implementation-Version the manifest of type's jar + * names, as the class loader read it when it defined type's package, + * when that manifest is keploy-sdk.jar's: its Implementation-Title is + * SDK_TITLE. It is "" when the manifest is another jar's (an app jar + * that shades the SDK's classes, or holds a copy of them), names no + * version, or type comes from no jar (a class dir). So of IndexReport, + * it is keploy-sdk.jar's version or "", never an app's. + */ + static String versionOf(Class type) { + Package pkg = type.getPackage(); + if (pkg == null || !SDK_TITLE.equals(pkg.getImplementationTitle())) { + return ""; + } + String version = pkg.getImplementationVersion(); + return version == null ? "" : version; } String toJson() { diff --git a/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java b/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java index e273bdbf..e6c26e44 100644 --- a/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java +++ b/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java @@ -263,9 +263,11 @@ private static boolean start(CoverageIndex index, Instrumentation instrumentatio private static void startJacoco(Instrumentation instrumentation, CoverageIndex index) { try { index.jacoco = IndexReport.JACOCO_NONE; - if (EmbeddedJacoco.attached(ClassLoader.getSystemClassLoader(), - ManagementFactory.getRuntimeMXBean().getInputArguments())) { + String attached = EmbeddedJacoco.attached(ClassLoader.getSystemClassLoader(), + ManagementFactory.getRuntimeMXBean().getInputArguments()); + if (attached != null) { index.jacoco = IndexReport.JACOCO_ATTACHED; + index.jacocoVersion = attached; log(Level.INFO, "Keploy dedup: this JVM has a JaCoCo agent attached, so Keploy reads coverage" + " from it, and it instruments every class its own options take in. Attach only" + " keploy-sdk.jar to have JaCoCo instrument the app's own classes alone", null); @@ -285,6 +287,7 @@ private static void startJacoco(Instrumentation instrumentation, CoverageIndex i String version = EmbeddedJacoco.start(instrumentation, EmbeddedJacoco.options(includes)); index.includes = includes; index.jacoco = IndexReport.JACOCO_EMBEDDED; + index.jacocoVersion = version; log(Level.INFO, "Keploy dedup: started JaCoCo " + version + (includes.isEmpty() ? " on every class" : " on the app's " + names.size() + " classes alone (includes=" @@ -1485,10 +1488,12 @@ private List entries() { private volatile String launch = ""; private volatile List packages = Collections.emptyList(); // What measures the classes, and the includes= of the JaCoCo agent - // the SDK started (IndexReport's JACOCO_*). Set by premain's - // startJacoco before the worker that sends the report starts. + // the SDK started (IndexReport's JACOCO_*), and that agent's + // version, or the attached one's. Set by premain's startJacoco + // before the worker that sends the report starts. private volatile String jacoco = IndexReport.JACOCO_UNCHECKED; private volatile String includes = ""; + private volatile String jacocoVersion = ""; private volatile IndexReport report; // report is the INDEX report of this index: computed once, on the @@ -1512,7 +1517,7 @@ private IndexReport report() { } } report = new IndexReport(IndexReport.buildId(ids), sourceKind, launch, roots, packages, - all.size(), includes, jacoco); + all.size(), includes, jacoco, IndexReport.versionOf(IndexReport.class), jacocoVersion); } return report; } diff --git a/keploy-sdk/src/test/java/io/keploy/dedup/AppIndexTest.java b/keploy-sdk/src/test/java/io/keploy/dedup/AppIndexTest.java index 7c568725..e79042ba 100644 --- a/keploy-sdk/src/test/java/io/keploy/dedup/AppIndexTest.java +++ b/keploy-sdk/src/test/java/io/keploy/dedup/AppIndexTest.java @@ -443,6 +443,9 @@ private static void assertIndex(String source, String launch, String packages, i // start() is not premain: it starts no JaCoCo and looks for none. assertEquals(index.toString(), "unchecked", index.get("jacoco").getAsString()); assertEquals(index.toString(), "", index.get("includes").getAsString()); + assertEquals(index.toString(), "", index.get("jacocoVersion").getAsString()); + // The SDK's classes come from the build's class dir here. + assertEquals(index.toString(), "", index.get("sdkVersion").getAsString()); } private static String sha256(String text) { diff --git a/keploy-sdk/src/test/java/io/keploy/dedup/EmbeddedJacocoTest.java b/keploy-sdk/src/test/java/io/keploy/dedup/EmbeddedJacocoTest.java index adf30542..6155da22 100644 --- a/keploy-sdk/src/test/java/io/keploy/dedup/EmbeddedJacocoTest.java +++ b/keploy-sdk/src/test/java/io/keploy/dedup/EmbeddedJacocoTest.java @@ -88,24 +88,81 @@ public void optionsCarryIncludesOnlyWhenThereAreAny() { public void aJacocoAgentOnTheClasspathOrAmongTheArgumentsCountsAsAttached() throws Exception { File jacoco = tmp.newFile("jacocoagent.jar"); AgentJar.extractTo(jacoco); + String version = jacocoVersion(); File other = sdkJar(); List none = Collections.emptyList(); try (URLClassLoader withJacoco = new URLClassLoader(new URL[] {jacoco.toURI().toURL()}, null); URLClassLoader without = new URLClassLoader(new URL[] {tmp.getRoot().toURI().toURL()}, null)) { // Attached before keploy-sdk.jar: on the classpath already. - assertTrue(EmbeddedJacoco.attached(withJacoco, none)); - assertFalse(EmbeddedJacoco.attached(without, none)); + assertEquals(version, EmbeddedJacoco.attached(withJacoco, none)); + assertNull(EmbeddedJacoco.attached(without, none)); // Attached after it: only its argument names it. - assertTrue(EmbeddedJacoco.attached(without, Arrays.asList("-Xmx64m", + assertEquals(version, EmbeddedJacoco.attached(without, Arrays.asList("-Xmx64m", "-javaagent:" + other.getPath(), "-javaagent:" + jacoco.getPath() + "=output=none"))); - assertTrue(EmbeddedJacoco.attached(without, Collections.singletonList("-javaagent:" + jacoco.getPath()))); + assertEquals(version, EmbeddedJacoco.attached(without, + Collections.singletonList("-javaagent:" + jacoco.getPath()))); // Another agent, a jar named as an option, a jar not there. - assertFalse(EmbeddedJacoco.attached(without, Arrays.asList("-javaagent:" + other.getPath(), + assertNull(EmbeddedJacoco.attached(without, Arrays.asList("-javaagent:" + other.getPath(), "-javaagent:" + other.getPath() + "=" + jacoco.getPath(), "-cp", jacoco.getPath(), "-javaagent:" + new File(tmp.getRoot(), "gone.jar").getPath()))); } } + @Test + public void aJacocoAgentWhoseJarNamesNoVersionIsAttachedWithNone() throws Exception { + // A JaCoCo runtime in a jar whose manifest names no version, and in + // a class dir: attached, at a version not known. + File jar = tmp.newFile("unversioned-jacoco.jar"); + Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(jar), manifest)) { + out.putNextEntry(new ZipEntry(EmbeddedJacoco.RUNTIME_CLASS)); + out.closeEntry(); + } + File dir = tmp.newFolder(); + File runtime = new File(dir, EmbeddedJacoco.RUNTIME_CLASS); + assertTrue(runtime.getParentFile().mkdirs()); + assertTrue(runtime.createNewFile()); + List none = Collections.emptyList(); + try (URLClassLoader inJar = new URLClassLoader(new URL[] {jar.toURI().toURL()}, null); + URLClassLoader inDir = new URLClassLoader(new URL[] {dir.toURI().toURL()}, null); + URLClassLoader without = new URLClassLoader(new URL[] {tmp.getRoot().toURI().toURL()}, null)) { + assertEquals("", EmbeddedJacoco.attached(inJar, none)); + assertEquals("", EmbeddedJacoco.attached(inDir, none)); + assertEquals("", EmbeddedJacoco.attached(without, + Collections.singletonList("-javaagent:" + jar.getPath()))); + } + } + + @Test + public void aJacocoRuntimeInAJarNestedInAnotherNamesNoVersion() throws Exception { + // A loader of its own may find the runtime in a jar nested in the + // app's: the outer jar's manifest is the app's, not JaCoCo's. + File app = tmp.newFile("app.jar"); + Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + manifest.getMainAttributes().put(Attributes.Name.IMPLEMENTATION_VERSION, "7.7.7-app"); + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(app), manifest)) { + out.putNextEntry(new ZipEntry("BOOT-INF/lib/jacocoagent.jar")); + out.closeEntry(); + } + String outer = "jar:" + app.toURI() + "!/"; + assertEquals("7.7.7-app", EmbeddedJacoco.versionOfJarHolding(new URL(outer + "BOOT-INF/classes/A.class"))); + assertEquals("", EmbeddedJacoco.versionOfJarHolding( + new URL(outer + "BOOT-INF/lib/jacocoagent.jar!/" + EmbeddedJacoco.RUNTIME_CLASS))); + // A runtime in no jar file. + assertEquals("", EmbeddedJacoco.versionOfJarHolding(new File(tmp.getRoot(), "A.class").toURI().toURL())); + } + + // jacocoVersion is the version the jar of the JaCoCo agent the SDK + // carries names: JaCoCo's own version, less its build qualifier. + private static String jacocoVersion() { + String qualified = org.jacoco.core.JaCoCo.VERSION; + String version = qualified.substring(0, qualified.lastIndexOf('.')); + assertTrue(qualified, version.matches("\\d+\\.\\d+\\.\\d+")); + return version; + } + @Test public void theSdkAloneStartsJacocoOnTheAppsClassesOnly() throws Exception { Run run = new Run(SDK); @@ -113,7 +170,8 @@ public void theSdkAloneStartsJacocoOnTheAppsClassesOnly() throws Exception { String output = runApp(run); // The INDEX report says what measures the app, and with what includes=. assertTrue(output, output.contains("\"source\":\"override\"")); - assertTrue(output, output.contains("\"classes\":1,\"includes\":\"org.example.embedded.*\",\"jacoco\":\"embedded\"}")); + assertTrue(output, output.contains("\"classes\":1,\"includes\":\"org.example.embedded.*\"," + + "\"jacoco\":\"embedded\",\"sdkVersion\":\"\",\"jacocoVersion\":\"" + jacocoVersion() + "\"}")); assertTrue(output, output.contains("agent=true")); assertTrue(output, output.contains("app=true")); // Loaded after JaCoCo started, outside its includes=. @@ -134,7 +192,10 @@ public void theIndexReportNamesAJacocoAgentTheCommandAttached() throws Exception Run run = new Run(SDK, JACOCO); run.env.put("EMBEDDED_APP_INDEX", "1"); String output = runApp(run); - assertTrue(output, output.contains("\"classes\":1,\"includes\":\"\",\"jacoco\":\"attached\"}")); + // The SDK's classes come from a class dir here, so it names no + // version of its own. + assertTrue(output, output.contains("\"classes\":1,\"includes\":\"\",\"jacoco\":\"attached\"," + + "\"sdkVersion\":\"\",\"jacocoVersion\":\"" + jacocoVersion() + "\"}")); } @Test @@ -154,6 +215,17 @@ public void aJacocoAgentAttachedBeforeTheSdkIsLeftToRunAlone() throws Exception assertLeftAlone(runApp(run)); } + @Test + public void theIndexReportNamesAJacocoAgentAttachedBeforeTheSdk() throws Exception { + // JaCoCo's jar is on the system classpath when the SDK's premain + // runs: its version is read from the jar holding its runtime. + Run run = new Run(JACOCO, SDK); + run.env.put("EMBEDDED_APP_INDEX", "1"); + String output = runApp(run); + assertTrue(output, output.contains("\"classes\":1,\"includes\":\"\",\"jacoco\":\"attached\"," + + "\"sdkVersion\":\"\",\"jacocoVersion\":\"" + jacocoVersion() + "\"}")); + } + @Test public void dedupTurnedOffStartsNoJacoco() throws Exception { Run run = new Run(SDK); @@ -178,7 +250,8 @@ public void aJacocoThatCannotStartLeavesTheAppRunningUnmeasured() throws Excepti assertTrue(output, output.contains("could not start the JaCoCo agent keploy-sdk.jar carries")); assertFalse(output, output.contains("started JaCoCo")); // The INDEX report says nothing measures the app. - assertTrue(output, output.contains("\"includes\":\"\",\"jacoco\":\"none\"}")); + assertTrue(output, output.contains( + "\"includes\":\"\",\"jacoco\":\"none\",\"sdkVersion\":\"\",\"jacocoVersion\":\"\"}")); } private static void assertLeftAlone(String output) { diff --git a/keploy-sdk/src/test/java/io/keploy/dedup/IndexReportTest.java b/keploy-sdk/src/test/java/io/keploy/dedup/IndexReportTest.java index 51ec8f72..06e3033f 100644 --- a/keploy-sdk/src/test/java/io/keploy/dedup/IndexReportTest.java +++ b/keploy-sdk/src/test/java/io/keploy/dedup/IndexReportTest.java @@ -2,19 +2,35 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.jar.Attributes; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import java.util.zip.ZipEntry; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; public class IndexReportTest { + @Rule + public final TemporaryFolder tmp = new TemporaryFolder(); + // Keploy's codecoverage.JavaBuildID of the ids 1, 0xff and // 0xffffffffffffffff (its TestBuildIDs): sha256 over // "0000000000000001\n00000000000000ff\nffffffffffffffff". @@ -59,7 +75,7 @@ public void aRootIsTheLocationWithoutTheClassPath() { public void theReportIsOneSmallJsonObjectWithNoClassList() { IndexReport report = new IndexReport(GO_BUILD_ID, IndexReport.SOURCE_DERIVED, "java -jar /app/app.jar", Arrays.asList("/app/libs/b.jar", "/app/classes", "/app/libs/b.jar"), - Collections.singletonList("org.acme"), 3, "org.acme.*", IndexReport.JACOCO_EMBEDDED); + Collections.singletonList("org.acme"), 3, "org.acme.*", IndexReport.JACOCO_EMBEDDED, "2.2.0", "0.8.15"); JsonObject json = JsonParser.parseString(report.toJson()).getAsJsonObject(); assertEquals("index", json.get("kind").getAsString()); assertEquals(1, json.get("v").getAsInt()); @@ -72,7 +88,12 @@ public void theReportIsOneSmallJsonObjectWithNoClassList() { assertEquals(3, json.get("classes").getAsInt()); assertEquals("org.acme.*", json.get("includes").getAsString()); assertEquals("embedded", json.get("jacoco").getAsString()); - assertEquals(11, json.size()); + assertEquals("2.2.0", json.get("sdkVersion").getAsString()); + assertEquals("0.8.15", json.get("jacocoVersion").getAsString()); + assertEquals(13, json.size()); + // The versions come last, after the fields an older SDK sent. + assertTrue(report.toJson(), report.toJson().endsWith( + "\"jacoco\":\"embedded\",\"sdkVersion\":\"2.2.0\",\"jacocoVersion\":\"0.8.15\"}")); // Its kind starts the object, so a reader can tell it from a test's // coverage record by its first bytes. assertEquals(0, report.toJson().indexOf("{\"kind\":\"index\",")); @@ -87,9 +108,72 @@ public void manyRootsAreCutAndCounted() { roots.add(String.format("/cp/lib-%03d.jar", i)); } IndexReport report = new IndexReport("", IndexReport.SOURCE_CLASSPATH, "why", roots, - Collections.emptyList(), 0, "", IndexReport.JACOCO_UNCHECKED); + Collections.emptyList(), 0, "", IndexReport.JACOCO_UNCHECKED, "", ""); assertEquals(IndexReport.MAX_ROOTS, report.roots.size()); assertEquals(IndexReport.MAX_ROOTS + 6, report.rootCount); assertEquals("/cp/lib-000.jar", report.roots.get(0)); } + + @Test + public void theSdkVersionIsWhatItsJarsManifestNames() throws Exception { + // As keploy-sdk.jar's build writes it: its title and the project's + // version. + assertEquals("2.2.0-row", versionOfMarkerIn(jarWithMarker(IndexReport.SDK_TITLE, "2.2.0-row"))); + // keploy-sdk.jar's manifest that names no version, and a class dir. + assertEquals("", versionOfMarkerIn(jarWithMarker(IndexReport.SDK_TITLE, null))); + File dir = tmp.newFolder(); + File copy = new File(dir, MARKER); + assertTrue(copy.getParentFile().mkdirs()); + try (InputStream input = Marker.class.getClassLoader().getResourceAsStream(MARKER)) { + Files.copy(input, copy.toPath()); + } + assertEquals("", versionOfMarkerIn(dir)); + // The SDK's own classes, here read from the build's class dir. + assertEquals("", IndexReport.versionOf(IndexReport.class)); + } + + @Test + public void anotherJarsVersionIsNeverTakenForTheSdks() throws Exception { + // An app's uber jar that shades the SDK's classes, or holds a copy + // of them, names the app's version: the SDK's is not known. + assertEquals("", versionOfMarkerIn(jarWithMarker("My App", "7.7.7-app"))); + assertEquals("", versionOfMarkerIn(jarWithMarker(null, "7.7.7-app"))); + } + + // Marker is a class versionOf is asked about, loaded from a jar or dir + // of the row's own, by a loader of its own. + static final class Marker { + } + + private static final String MARKER = Marker.class.getName().replace('.', '/') + ".class"; + + private File jarWithMarker(String title, String version) throws Exception { + File jar = tmp.newFile("marker-" + System.nanoTime() + ".jar"); + Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + if (title != null) { + manifest.getMainAttributes().put(Attributes.Name.IMPLEMENTATION_TITLE, title); + } + if (version != null) { + manifest.getMainAttributes().put(Attributes.Name.IMPLEMENTATION_VERSION, version); + } + try (JarOutputStream out = new JarOutputStream(new FileOutputStream(jar), manifest); + InputStream input = Marker.class.getClassLoader().getResourceAsStream(MARKER)) { + out.putNextEntry(new ZipEntry(MARKER)); + byte[] buffer = new byte[4096]; + for (int read; (read = input.read(buffer)) != -1; ) { + out.write(buffer, 0, read); + } + out.closeEntry(); + } + return jar; + } + + private static String versionOfMarkerIn(File from) throws Exception { + try (URLClassLoader loader = new URLClassLoader(new URL[] {from.toURI().toURL()}, null)) { + Class marker = Class.forName(Marker.class.getName(), false, loader); + assertTrue("loaded from " + from, marker != Marker.class); + return IndexReport.versionOf(marker); + } + } } diff --git a/scripts/smoke-javaagent.sh b/scripts/smoke-javaagent.sh index 9716be29..274d64d3 100755 --- a/scripts/smoke-javaagent.sh +++ b/scripts/smoke-javaagent.sh @@ -68,6 +68,9 @@ import java.nio.charset.StandardCharsets; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.jar.JarFile; +import java.util.jar.JarInputStream; +import java.util.jar.Manifest; public final class SmokeHarness { private static final File CONTROL_SOCKET = new File("/tmp/coverage_control.sock"); @@ -150,6 +153,20 @@ public final class SmokeHarness { + wantSource + ", jacoco " + wantJacoco + (wantClasses.isEmpty() ? "" : " and " + wantClasses + " classes") + " in " + mode + ": " + report); } + // keploy-sdk's version and the measuring JaCoCo's, as their jars' + // manifests name them: the one the command attached, or the one + // keploy-sdk.jar carries and started. + String wantSdkVersion = versionOf(new File(System.getProperty("smoke.sdkJar")), null); + String wantJacocoVersion = "attached".equals(wantJacoco) + ? versionOf(new File(System.getProperty("smoke.jacocoJar")), null) + : "embedded".equals(wantJacoco) + ? versionOf(new File(System.getProperty("smoke.sdkJar")), "jacocoagent.jar") : ""; + if (wantSdkVersion.isEmpty() || wantJacocoVersion.isEmpty() != !wantJacoco.matches("attached|embedded") + || !report.endsWith(",\"sdkVersion\":\"" + wantSdkVersion + "\",\"jacocoVersion\":\"" + + wantJacocoVersion + "\"}")) { + throw new IllegalStateException("the INDEX report did not name keploy-sdk " + wantSdkVersion + + " and JaCoCo " + wantJacocoVersion + " in " + mode + ": " + report); + } String ownLibraries = System.getProperty("smoke.ownLibraries", ""); if (!ownLibraries.isEmpty()) { loadsItsOwn(ownLibraries); @@ -205,6 +222,21 @@ public final class SmokeHarness { } } + // versionOf is the Implementation-Version of jar's manifest, or of the + // manifest of the jar nested in it at entry when entry is not null. + private static String versionOf(File jar, String entry) throws Exception { + try (JarFile file = new JarFile(jar)) { + Manifest manifest = file.getManifest(); + if (entry != null) { + try (JarInputStream nested = new JarInputStream(file.getInputStream(file.getEntry(entry)))) { + manifest = nested.getManifest(); + } + } + String version = manifest == null ? null : manifest.getMainAttributes().getValue("Implementation-Version"); + return version == null ? "" : version; + } + } + private static void waitFor(File file) throws InterruptedException { long deadline = System.currentTimeMillis() + 10000L; while (System.currentTimeMillis() < deadline) { @@ -268,6 +300,7 @@ java_with_agents() { java \ -javaagent:"${agent_jar}" \ -javaagent:"${jacoco_jar}=destfile=/tmp/keploy-sdk-smoke-${mode}.exec" \ + -Dsmoke.sdkJar="${agent_jar}" -Dsmoke.jacocoJar="${jacoco_jar}" \ "$@" } @@ -321,7 +354,8 @@ jar cfm "${work_dir}/jar/plain-app.jar" "${work_dir}/jar/MANIFEST.MF" -C "${work java_with_sdk_only() { local mode="${1:?mode required}" shift - java -javaagent:"${agent_jar}" "$@" 2>"${work_dir}/${mode}.err" || { cat "${work_dir}/${mode}.err" >&2; exit 1; } + java -javaagent:"${agent_jar}" -Dsmoke.sdkJar="${agent_jar}" "$@" 2>"${work_dir}/${mode}.err" \ + || { cat "${work_dir}/${mode}.err" >&2; exit 1; } if ! grep -q "started JaCoCo .* on the app's .* classes alone (includes=" "${work_dir}/${mode}.err"; then cat "${work_dir}/${mode}.err" >&2 echo "${mode}: keploy-sdk.jar did not start its JaCoCo agent on the app's classes" >&2