Keep a javadoc {@snippet} as it is written

With javadoc formatting on, the lines of a {@snippet ...} tag were
reflowed like prose: the code inside came out on one line, "<p>" in
front of it, and a "//" comment in the snippet swallowed the closing
brace. The lexer treated the snippet as any other inline tag, whose
contents are joined and wrapped.

This ports google/google-java-format#1141 by cpovirk: "{@snippet"
that starts an inline tag is its own token, everything up to its
closing brace keeps its newlines and spaces as inside <pre>, and the
writer puts a blank line before and after the snippet. Nothing inside
the snippet is reformatted. The two tests come from upstream; a third
one pins a snippet with indentation and a // markup comment.

Adapted to this lexer, which keeps inline tags as INLINE_TAG_OPEN and
INLINE_TAG_CLOSE tokens: a "{@snippet" inside another inline tag is an
ordinary inline tag, as upstream's is an ordinary literal.

Javadoc formatting is off unless JavaFormatterOptions asks for it, so
the default output is unchanged.

Author: cpovirk

open-java-format/src/main/java/com/palantir/javaformat/java/javadoc/JavadocFormatter.java

@@ -69,6 +69,12 @@ private String render(List<Token> input, int blockIndent) {
                 case FOOTER_JAVADOC_TAG_START:
                     output.writeFooterJavadocTagStart(token);
                     break;
+                case SNIPPET_BEGIN:
+                    output.writeSnippetBegin(token);
+                    break;
+                case SNIPPET_END:
+                    output.writeSnippetEnd(token);
+                    break;
                 case LIST_OPEN_TAG:
                     output.writeListOpen(token);
                     break;

open-java-format/src/main/java/com/palantir/javaformat/java/javadoc/JavadocLexer.java

@@ -44,6 +44,8 @@
 import static com.palantir.javaformat.java.javadoc.Token.Type.PARAGRAPH_OPEN_TAG;
 import static com.palantir.javaformat.java.javadoc.Token.Type.PRE_CLOSE_TAG;
 import static com.palantir.javaformat.java.javadoc.Token.Type.PRE_OPEN_TAG;
+import static com.palantir.javaformat.java.javadoc.Token.Type.SNIPPET_BEGIN;
+import static com.palantir.javaformat.java.javadoc.Token.Type.SNIPPET_END;
 import static com.palantir.javaformat.java.javadoc.Token.Type.TABLE_CLOSE_TAG;
 import static com.palantir.javaformat.java.javadoc.Token.Type.TABLE_OPEN_TAG;
 import static com.palantir.javaformat.java.javadoc.Token.Type.WHITESPACE;
@@ -98,6 +100,7 @@ private static String stripJavadocBeginAndEnd(String input) {
     private final NestingCounter preDepth = new NestingCounter();
     private final NestingCounter codeDepth = new NestingCounter();
     private final NestingCounter tableDepth = new NestingCounter();
+    private boolean outerInlineTagIsSnippet;
     private boolean somethingSinceNewline;
 
     private JavadocLexer(CharStream input) {
@@ -159,13 +162,26 @@ private Token.Type consumeToken() throws LexException {
         }
         somethingSinceNewline = true;
 
-        if (input.tryConsumeRegex(INLINE_TAG_OPEN_PATTERN)) {
+        if (input.tryConsumeRegex(SNIPPET_TAG_OPEN_PATTERN)) {
+            if (braceDepth.value() == 0) {
+                braceDepth.increment();
+                outerInlineTagIsSnippet = true;
+                return SNIPPET_BEGIN;
+            }
+            braceDepth.increment();
+            return INLINE_TAG_OPEN;
+        } else if (input.tryConsumeRegex(INLINE_TAG_OPEN_PATTERN)) {
             braceDepth.increment();
             return INLINE_TAG_OPEN;
         } else if (input.tryConsume("{")) {
             braceDepth.incrementIfPositive();
             return LITERAL;
         } else if (input.tryConsume("}")) {
+            if (outerInlineTagIsSnippet && braceDepth.value() == 1) {
+                braceDepth.decrementIfPositive();
+                outerInlineTagIsSnippet = false;
+                return SNIPPET_END;
+            }
             braceDepth.decrementIfPositive();
             return braceDepth.isPositive() ? LITERAL : INLINE_TAG_CLOSE;
         }
@@ -240,7 +256,7 @@ private Token.Type consumeToken() throws LexException {
     }
 
     private boolean preserveExistingFormatting() {
-        return preDepth.isPositive() || tableDepth.isPositive() || codeDepth.isPositive();
+        return preDepth.isPositive() || tableDepth.isPositive() || codeDepth.isPositive() || outerInlineTagIsSnippet;
     }
 
     private void checkMatchingTags() throws LexException {
@@ -541,6 +557,7 @@ private static boolean hasMultipleNewlines(String s) {
     private static final Pattern BLOCKQUOTE_OPEN_PATTERN = openTagPattern("blockquote");
     private static final Pattern BLOCKQUOTE_CLOSE_PATTERN = closeTagPattern("blockquote");
     private static final Pattern BR_PATTERN = openTagPattern("br");
+    private static final Pattern SNIPPET_TAG_OPEN_PATTERN = compile("^[{]@snippet\\b");
     private static final Pattern INLINE_TAG_OPEN_PATTERN = compile("^[{]@\\w*");
     /*
      * We exclude < so that we don't swallow following HTML tags. This lets us fix up "foo<p>" (~400

open-java-format/src/main/java/com/palantir/javaformat/java/javadoc/JavadocWriter.java

@@ -126,6 +126,30 @@ void writeFooterJavadocTagStart(Token token) {
         continuingFooterTag = true;
     }
 
+    void writeSnippetBegin(Token token) {
+        requestBlankLine();
+        writeToken(token);
+        /*
+         * We don't request a newline here because we should have at least a colon following on this
+         * line, and we may have attributes after that.
+         *
+         * (If we find it convenient, we could instead consume the entire rest of the line as part of
+         * the same token as `{@snippet` itself. But we already would never split the rest of the line
+         * across lines (because we preserve whitespace), so that might not accomplish anything. Plus,
+         * we'd probably want to be careful not to swallow an expectedly early closing `}`.)
+         */
+    }
+
+    void writeSnippetEnd(Token token) {
+        /*
+         * We don't request a newline here because we have preserved all newlines that existed in the
+         * input. Specifically, if there is not yet a newline, one could be added; several could be
+         * collapsed; and a closing brace that is not indented as we'd want could be indented.
+         */
+        writeToken(token);
+        requestBlankLine();
+    }
+
     void writeListOpen(Token token) {
         requestBlankLine();
 

open-java-format/src/main/java/com/palantir/javaformat/java/javadoc/Token.java

@@ -39,6 +39,10 @@ enum Type {
         END_JAVADOC,
         /** The {@code @foo} that begins a block Javadoc tag like {@code @throws}. */
         FOOTER_JAVADOC_TAG_START,
+        /** The opening {@code ｛@snippet} of a code snippet. */
+        SNIPPET_BEGIN,
+        /** The closing {@code ｝} of a code snippet. */
+        SNIPPET_END,
         LIST_OPEN_TAG,
         LIST_CLOSE_TAG,
         LIST_ITEM_OPEN_TAG,

open-java-format/src/test/java/com/palantir/javaformat/java/JavadocFormattingTest.java

@@ -1418,4 +1418,65 @@ public void u2028LineSeparator() {
         };
         doFormatTest(input, expected);
     }
+
+    @Test
+    public void blankLinesAroundSnippetAndNoMangling() {
+        String[] input = {
+            "/**", //
+            " * hello world",
+            " * {@snippet :",
+            " * public class Foo {",
+            " *   private String s;",
+            " * }",
+            " * }",
+            " * hello again",
+            " */",
+            "class Test {}",
+        };
+        String[] expected = {
+            "/**", //
+            " * hello world",
+            " *",
+            " * {@snippet :",
+            " * public class Foo {",
+            " *   private String s;",
+            " * }",
+            " * }",
+            " *",
+            " * hello again",
+            " */",
+            "class Test {}",
+        };
+        doFormatTest(input, expected);
+    }
+
+    @Test
+    public void notASnippetUnlessOuterTag() {
+        String[] input = {
+            "/** I would like to tell you about the {@code {@snippet ...}} tag. */", "class Test {}",
+        };
+        String[] expected = {
+            "/** I would like to tell you about the {@code {@snippet ...}} tag. */", "class Test {}",
+        };
+        doFormatTest(input, expected);
+    }
+
+    @Test
+    public void snippetKeepsCommentsAndIndentation() {
+        String[] input = {
+            "/**", //
+            " * Example usage:",
+            " *",
+            " * {@snippet :",
+            " *   int x = 1;",
+            " *   foo(x,",
+            " *       y);   // @highlight substring=\"foo\"",
+            " * }",
+            " *",
+            " * Done.",
+            " */",
+            "class Test {}",
+        };
+        doFormatTest(input, input);
+    }
 }