diff --git a/embedding/commentfilter/c_style_filter_test.go b/embedding/commentfilter/c_style_filter_test.go index 850d152f..c1314ccd 100644 --- a/embedding/commentfilter/c_style_filter_test.go +++ b/embedding/commentfilter/c_style_filter_test.go @@ -139,4 +139,31 @@ var _ = Describe("C and C++", func() { assertFiltered("sample.cpp", RetainNone, lines, expected) }) + + It("should preserve escaped and unterminated quoted literals", func() { + lines := []string{ + `const char* escaped = "quote: \\""; // comment`, + `const char* open = "unterminated`, + `const char* invalid = R"bad delimiter(text)bad delimiter";`, + `const char* missing = R"delimiter`, + } + expected := []string{ + `const char* escaped = "quote: \\""; // comment`, + `const char* open = "unterminated`, + `const char* invalid = R"bad delimiter(text)bad delimiter";`, + `const char* missing = R"delimiter`, + } + + assertFiltered("sample.cpp", RetainNone, lines, expected) + }) + + It("should keep an open block comment across lines", func() { + lines := []string{ + "/* open block", + "continued", + "closed */ int value = 1;", + } + + assertFiltered("sample.cpp", RetainBlock, lines, lines) + }) }) diff --git a/embedding/commentfilter/csharp_filter_test.go b/embedding/commentfilter/csharp_filter_test.go index 14fbea41..f51cf14e 100644 --- a/embedding/commentfilter/csharp_filter_test.go +++ b/embedding/commentfilter/csharp_filter_test.go @@ -165,4 +165,31 @@ var _ = Describe("C#", func() { assertFiltered("Api.cs", RetainNone, lines, expected) }) + + It("should preserve C# string variants inside interpolation expressions", func() { + cases := [][]string{ + {`var value = "quote: \\"";`}, + {`var value = $"{Format("a\\\"b", @"a "" b", """raw""")}";`}, + {`var value = $"{Format("""raw`}, + {`var value = $"{Format("text`}, + {`var value = $"{number:000`}, + {`var value = $"""Keep {{ and }} {number}""";`}, + {`var value = @"a "" b";`}, + {`var value = "unterminated`}, + } + + for _, lines := range cases { + assertFiltered("Api.cs", RetainNone, lines, lines) + } + }) + + It("should keep an open block comment across lines", func() { + lines := []string{ + "/* open block", + "continued", + "closed */ var value = 1;", + } + + assertFiltered("Api.cs", RetainBlock, lines, lines) + }) }) diff --git a/embedding/commentfilter/filter_test.go b/embedding/commentfilter/filter_test.go index 80158f1f..5ae7247b 100644 --- a/embedding/commentfilter/filter_test.go +++ b/embedding/commentfilter/filter_test.go @@ -44,6 +44,19 @@ func TestCommentFilter(t *testing.T) { } var _ = Describe("Comment filter", func() { + It("should parse supported comment modes", func() { + mode, err := ParseMode("") + Expect(err).ShouldNot(HaveOccurred()) + Expect(mode).Should(Equal(RetainAll)) + + mode, err = ParseMode(string(RetainBlock)) + Expect(err).ShouldNot(HaveOccurred()) + Expect(mode).Should(Equal(RetainBlock)) + + _, err = ParseMode("unsupported") + Expect(err).Should(HaveOccurred()) + }) + Describe("unsupported extensions", func() { It("should return unsupported files unchanged", func() { lines := []string{ diff --git a/embedding/commentfilter/java_style_filter_test.go b/embedding/commentfilter/java_style_filter_test.go index 941fd243..cb86cfd0 100644 --- a/embedding/commentfilter/java_style_filter_test.go +++ b/embedding/commentfilter/java_style_filter_test.go @@ -30,6 +30,7 @@ import ( . "embed-code/embed-code-go/embedding/commentfilter" . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) var _ = Describe("Java-style languages", func() { @@ -120,4 +121,15 @@ var _ = Describe("Java-style languages", func() { assertFiltered("Api.java", RetainNone, lines, expected) }) + + It("should support configured documentation line markers", func() { + filter := MarkerCommentFilter{Syntax: CommentMarker{ + Documentation: DocumentationMarker{Inline: []string{"///"}}, + }} + lines := []string{"/// docs", "code"} + + result := filter.Filter(lines, RetainDocumentation) + + Expect(result).Should(Equal(lines)) + }) }) diff --git a/embedding/commentfilter/javascript_filter_test.go b/embedding/commentfilter/javascript_filter_test.go index 372599cd..66e92bcc 100644 --- a/embedding/commentfilter/javascript_filter_test.go +++ b/embedding/commentfilter/javascript_filter_test.go @@ -193,4 +193,31 @@ var _ = Describe("JavaScript and TypeScript", func() { assertFiltered("sample.ts", RetainNone, lines, expected) }) + + It("should preserve JavaScript regex, string, and interpolation variants", func() { + lines := []string{ + `const text = "escaped \\" // literal"; // comment`, + `const classPattern = /[\\/]/gi; // comment`, + `/start/.test(value); // comment`, + `const open = /unterminated`, + `const division = identifier / 2; // comment`, + "const template = `escaped \\` text ${value}`; // comment", + "const nested = `${{ value: /[}]/g, text: \"}\" }}`; // comment", + `const unusual = */ /regex/; // comment`, + `*/ /regex/; // comment`, + } + expected := []string{ + `const text = "escaped \\" `, + `const classPattern = /[\\/]/gi; `, + `/start/.test(value); `, + `const open = /unterminated`, + `const division = identifier / 2; `, + "const template = `escaped \\` text ${value}`; ", + "const nested = `${{ value: /[}]/g, text: \"}\" }}`; ", + `const unusual = */ /regex/; `, + `*/ /regex/; `, + } + + assertFiltered("sample.ts", RetainNone, lines, expected) + }) }) diff --git a/embedding/commentfilter/kotlin_filter_test.go b/embedding/commentfilter/kotlin_filter_test.go index 66e729d7..67ee0031 100644 --- a/embedding/commentfilter/kotlin_filter_test.go +++ b/embedding/commentfilter/kotlin_filter_test.go @@ -168,4 +168,23 @@ var _ = Describe("Kotlin", func() { assertFiltered("Sample.kt", RetainNone, lines, expected) }) + + It("should preserve escaped strings, characters, and nested interpolation braces", func() { + lines := []string{ + `val quote = '\'' // comment`, + `val text = "escaped \\" // literal" // comment`, + `val nested = "${if (ready) { "// literal" } else { value }}" // comment`, + `val raw = "${"""raw"""}" // comment`, + `val quoted = "${"text"}" // comment`, + } + expected := []string{ + `val quote = '\'' `, + `val text = "escaped \\" `, + `val nested = "${if (ready) { "// literal" } else { value }}" `, + `val raw = "${"""raw"""}" `, + `val quoted = "${"text"}" `, + } + + assertFiltered("Sample.kt", RetainNone, lines, expected) + }) }) diff --git a/embedding/commentfilter/python_filter_test.go b/embedding/commentfilter/python_filter_test.go index c34f48ef..2fab2926 100644 --- a/embedding/commentfilter/python_filter_test.go +++ b/embedding/commentfilter/python_filter_test.go @@ -120,4 +120,42 @@ var _ = Describe("Python", func() { assertFiltered("module.py", RetainNone, lines, expected) }) + + It("should preserve strings nested inside f-string expressions", func() { + cases := []struct { + lines []string + expected []string + }{ + { + lines: []string{`value = f"{format('escaped \\' quote')}" # comment`}, + expected: []string{`value = f"{format('escaped \\' quote')}" # comment`}, + }, + { + lines: []string{`value = f"{format("""raw`, `text`, `""")}" # comment`}, + expected: []string{ + `value = f"{format("""raw`, `text`, `""")}" `, + }, + }, + { + lines: []string{`value = f"{number:04`}, + expected: []string{`value = f"{number:04`}, + }, + { + lines: []string{`value = f"{format('unterminated`}, + expected: []string{`value = f"{format('unterminated`}, + }, + { + lines: []string{`value = f"{ {'nested': value} }" # comment`}, + expected: []string{`value = f"{ {'nested': value} }" `}, + }, + { + lines: []string{`value = "unterminated`}, + expected: []string{`value = "unterminated`}, + }, + } + + for _, tc := range cases { + assertFiltered("module.py", RetainNone, tc.lines, tc.expected) + } + }) }) diff --git a/embedding/commentfilter/visual_basic_filter_test.go b/embedding/commentfilter/visual_basic_filter_test.go index d28ee970..4d3c65cc 100644 --- a/embedding/commentfilter/visual_basic_filter_test.go +++ b/embedding/commentfilter/visual_basic_filter_test.go @@ -91,4 +91,10 @@ var _ = Describe("Visual Basic", func() { assertFiltered("Module.vb", RetainDocumentation, lines, expected) }) + + It("should preserve an unterminated quoted string", func() { + lines := []string{`Dim value = "unterminated`} + + assertFiltered("Module.vb", RetainNone, lines, lines) + }) }) diff --git a/embedding/embedding_test.go b/embedding/embedding_test.go index 77dd3839..685fc24f 100644 --- a/embedding/embedding_test.go +++ b/embedding/embedding_test.go @@ -360,6 +360,131 @@ var _ = Describe("Embedding", func() { Expect(context.IsContainsEmbedding()).Should(BeFalse()) Expect(processor.IsUpToDate()).Should(BeTrue()) }) + + It("should report an invalid include pattern", func() { + config := configuration.NewConfiguration() + config.DocumentationRoot = GinkgoT().TempDir() + config.DocIncludes = []string{"["} + + _, err := embedding.NewProcessor("unused.md", config) + + Expect(err).Should(HaveOccurred()) + + _, err = embedding.EmbedAll(config) + + Expect(err).Should(HaveOccurred()) + }) + + It("should report an invalid exclude pattern", func() { + config := configuration.NewConfiguration() + config.DocumentationRoot = GinkgoT().TempDir() + config.DocIncludes = nil + config.DocExcludes = []string{"["} + + _, err := embedding.NewProcessor("unused.md", config) + + Expect(err).Should(HaveOccurred()) + }) + + It("should return an empty result when embedding fails", func() { + documentationRoot := GinkgoT().TempDir() + config := configuration.NewConfiguration() + config.DocumentationRoot = documentationRoot + config.DocIncludes = []string{"*.md"} + Expect(os.WriteFile( + filepath.Join(documentationRoot, "invalid.md"), + []byte("\n"), + 0600, + )).To(Succeed()) + + result, err := embedding.EmbedAll(config) + + Expect(err).Should(HaveOccurred()) + Expect(result).Should(Equal(embedding.EmbedAllResult{})) + }) + + It("should return an empty named embedding result when no documents match", func() { + config := configuration.NewConfiguration() + config.Name = "empty" + config.DocumentationRoot = GinkgoT().TempDir() + config.DocIncludes = nil + + result, err := embedding.EmbedAll(config) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(result).Should(Equal(embedding.EmbedAllResult{})) + }) + + It("should return a document write error", func() { + documentationRoot := GinkgoT().TempDir() + docPath := filepath.ToSlash(filepath.Join(documentationRoot, "doc.md")) + config := configuration.NewConfiguration() + config.DocumentationRoot = documentationRoot + config.DocIncludes = []string{"doc.md"} + Expect(os.WriteFile(docPath, []byte("content"), 0600)).To(Succeed()) + processor, err := embedding.NewProcessor(docPath, config) + Expect(err).ShouldNot(HaveOccurred()) + state := writeFailureState{path: docPath} + processor.TransitionsMap = parsing.TransitionMap{ + parsing.Start: []parsing.State{state}, + state: []parsing.State{parsing.Finish}, + } + + context, err := processor.Embed() + + Expect(err).Should(HaveOccurred()) + Expect(context).ShouldNot(BeNil()) + }) + + It("should report an unreadable selected document as not up to date", func() { + documentationRoot := GinkgoT().TempDir() + docPath := filepath.ToSlash(filepath.Join(documentationRoot, "removed.md")) + config := configuration.NewConfiguration() + config.DocumentationRoot = documentationRoot + config.DocIncludes = []string{"removed.md"} + Expect(os.WriteFile(docPath, []byte("content"), 0600)).To(Succeed()) + processor, err := embedding.NewProcessor(docPath, config) + Expect(err).ShouldNot(HaveOccurred()) + Expect(os.Remove(docPath)).To(Succeed()) + + Expect(processor.IsUpToDate()).Should(BeFalse()) + }) + + It("should select fallback error lines from parser context", func() { + documentationRoot := GinkgoT().TempDir() + docPath := filepath.ToSlash(filepath.Join(documentationRoot, "doc.md")) + config := configuration.NewConfiguration() + config.DocumentationRoot = documentationRoot + config.DocIncludes = []string{"doc.md"} + Expect(os.WriteFile(docPath, []byte("first\nsecond"), 0600)).To(Succeed()) + processor, err := embedding.NewProcessor(docPath, config) + Expect(err).ShouldNot(HaveOccurred()) + processor.TransitionsMap = parsing.TransitionMap{ + parsing.Start: []parsing.State{failingEmbeddingState{}}, + } + + _, err = processor.Embed() + + var processingErr embedding.ProcessingError + Expect(errors.As(err, &processingErr)).Should(BeTrue()) + Expect(processingErr.Line).Should(Equal(1)) + }) + + It("should report an unexpected transition without an active embedding", func() { + documentationRoot := GinkgoT().TempDir() + docPath := filepath.ToSlash(filepath.Join(documentationRoot, "doc.md")) + config := configuration.NewConfiguration() + config.DocumentationRoot = documentationRoot + config.DocIncludes = []string{"doc.md"} + Expect(os.WriteFile(docPath, []byte("content"), 0600)).To(Succeed()) + processor, err := embedding.NewProcessor(docPath, config) + Expect(err).ShouldNot(HaveOccurred()) + processor.TransitionsMap = parsing.TransitionMap{} + + _, err = processor.Embed() + + Expect(err).Should(MatchError(ContainSubstring("unexpected parser state at line 1"))) + }) }) // buildConfigWithSourceFiles builds an embedding config with an isolated documentation root. @@ -386,3 +511,52 @@ func newProcessor( return processor } + +// failingEmbeddingState starts an embedding and returns a generic parser error. +type failingEmbeddingState struct{} + +// Accept starts an embedding at the second source line and fails. +func (failingEmbeddingState) Accept( + context *parsing.Context, + _ configuration.Configuration, +) error { + context.StartEmbedding(parsing.Instruction{}) + context.ToNextLine() + context.ToNextLine() + context.SetCodeStart() + + return errors.New("failure") +} + +// Recognize accepts every parser context. +func (failingEmbeddingState) Recognize(parsing.Context) bool { + return true +} + +// writeFailureState replaces the parsed document with a directory before it is written. +type writeFailureState struct { + // path is the document path to replace. + path string +} + +// Accept produces changed embedding content and makes the document path unwritable as a file. +func (s writeFailureState) Accept( + context *parsing.Context, + _ configuration.Configuration, +) error { + context.StartEmbedding(parsing.Instruction{}) + context.Result = append(context.Result, "changed") + for !context.ReachedEOF() { + context.ToNextLine() + } + context.FinishEmbedding() + Expect(os.Remove(s.path)).To(Succeed()) + Expect(os.Mkdir(s.path, 0700)).To(Succeed()) + + return nil +} + +// Recognize accepts every parser context. +func (writeFailureState) Recognize(parsing.Context) bool { + return true +} diff --git a/embedding/orchestration.go b/embedding/orchestration.go index cc2bdeec..37ff8ace 100644 --- a/embedding/orchestration.go +++ b/embedding/orchestration.go @@ -35,7 +35,6 @@ import ( "embed-code/embed-code-go/configuration" "embed-code/embed-code-go/embedding/parsing" - "embed-code/embed-code-go/fragmentation" "embed-code/embed-code-go/logging" "github.com/bmatcuk/doublestar/v4" @@ -167,10 +166,7 @@ func processRequiredDocs( } var processingErrors []error - resolver, err := fragmentation.NewResolver(fragmentation.DefaultResolverCacheLimit) - if err != nil { - return requiredDocPaths, []error{err} - } + resolver := newDefaultResolver() for _, doc := range requiredDocPaths { processor := newProcessor(doc, config, parsing.Transitions, requiredDocPaths, resolver) if err := handle(doc, processor); err != nil { diff --git a/embedding/parsing/instruction_test.go b/embedding/parsing/instruction_test.go index ac0ffc17..cbff75cf 100644 --- a/embedding/parsing/instruction_test.go +++ b/embedding/parsing/instruction_test.go @@ -182,6 +182,45 @@ var _ = Describe("Instruction", func() { Expect(err).Should(MatchError(ContainSubstring("invalid start pattern `[`"))) }) + It("should report invalid end and line patterns", func() { + for _, params := range []TestInstructionParams{ + {endGlob: "["}, + {lineGlob: "["}, + } { + xmlString := buildInstruction("org/example/Hello.java", params) + + _, err := parsing.FromXML(xmlString, config) + + Expect(err).Should(HaveOccurred()) + } + }) + + It("should expose instruction and pattern diagnostic strings", func() { + pattern, err := parsing.NewPattern("class") + Expect(err).ShouldNot(HaveOccurred()) + instruction := parsing.Instruction{ + CodeFile: "Example.java", + StartPattern: &pattern, + } + + Expect(pattern.String()).Should(Equal("Pattern class")) + Expect(instruction.String()).Should(ContainSubstring("file=`Example.java`")) + Expect(instruction.String()).Should(ContainSubstring("start=`Pattern class`")) + }) + + It("should not search before the first source line", func() { + pattern, err := parsing.NewPattern("class") + Expect(err).ShouldNot(HaveOccurred()) + + _, _, found := pattern.FindIn([]string{"class Example"}, -1) + Expect(found).Should(BeFalse()) + }) + + It("should not search with an empty pattern", func() { + _, _, found := (parsing.Pattern{}).FindIn([]string{"class Example"}, 0) + Expect(found).Should(BeFalse()) + }) + It("should embed a line ending with a literal opening brace pattern", func() { instructionParams := TestInstructionParams{ lineGlob: "class Hello {", diff --git a/embedding/parsing/state_test.go b/embedding/parsing/state_test.go index 3301724f..dd9d92fb 100644 --- a/embedding/parsing/state_test.go +++ b/embedding/parsing/state_test.go @@ -41,6 +41,78 @@ import ( ) var _ = Describe("Parser states", func() { + It("should accept parser boundary states", func() { + config := configuration.NewConfiguration() + context := newStateContext() + + Expect(parsing.Start.Recognize(context)).Should(BeTrue()) + Expect(parsing.Start.Accept(&context, config)).Should(Succeed()) + context.ToNextLine() + Expect(parsing.Finish.Recognize(context)).Should(BeTrue()) + Expect(parsing.Finish.Accept(&context, config)).Should(Succeed()) + }) + + It("should reject an embedding fence end when no fence marker is active", func() { + context := newStateContext("```") + + Expect(parsing.CodeFenceEnd.Recognize(context)).Should(BeFalse()) + + context.CodeFenceStarted = true + Expect(parsing.CodeFenceEnd.Recognize(context)).Should(BeFalse()) + }) + + It("should open and close an ordinary Markdown fence with a matching marker", func() { + config := configuration.NewConfiguration() + context := newStateContext("````", "````") + + Expect(parsing.RegularLine.Accept(&context, config)).Should(Succeed()) + Expect(context.MarkdownFenceStarted).Should(BeTrue()) + Expect(context.MarkdownFenceMarker).Should(Equal("````")) + + Expect(parsing.RegularLine.Accept(&context, config)).Should(Succeed()) + Expect(context.MarkdownFenceStarted).Should(BeFalse()) + Expect(context.MarkdownFenceMarker).Should(BeEmpty()) + }) + + DescribeTable("should keep an ordinary Markdown fence open", + func(line string, marker string, indentation int) { + config := configuration.NewConfiguration() + context := newStateContext(line) + context.MarkdownFenceStarted = true + context.MarkdownFenceMarker = marker + context.MarkdownFenceIndentation = indentation + + Expect(parsing.RegularLine.Accept(&context, config)).Should(Succeed()) + + Expect(context.MarkdownFenceStarted).Should(BeTrue()) + Expect(context.MarkdownFenceMarker).Should(Equal(marker)) + Expect(context.MarkdownFenceIndentation).Should(Equal(indentation)) + }, + Entry("when the closing fence indentation differs", " ```", "```", 0), + Entry("when the closing fence marker is shorter", "```", "````", 0), + Entry("when the closing fence marker differs", "```", "~~~~", 0), + Entry("when the closing fence has an info string", "``` language", "```", 0), + ) + + It("should ignore ordinary Markdown fences while processing an embedding", func() { + config := configuration.NewConfiguration() + context := newStateContext("```") + context.StartEmbedding(parsing.Instruction{}) + + Expect(parsing.RegularLine.Accept(&context, config)).Should(Succeed()) + + Expect(context.MarkdownFenceStarted).Should(BeFalse()) + Expect(context.MarkdownFenceMarker).Should(BeEmpty()) + Expect(context.MarkdownFenceIndentation).Should(BeZero()) + }) + + It("should expose parser diagnostic strings", func() { + context := newStateContext("line") + + Expect(context.String()).Should(ContainSubstring("file=`")) + Expect(context.String()).Should(ContainSubstring("line=`0`")) + }) + It("should consume a multiline instruction before opening an embedding fence", func() { config := configuration.NewConfiguration() context := newStateContext( @@ -172,6 +244,24 @@ var _ = Describe("Parser states", func() { )) }) + It("should stop an unclosed opening tag at a closing or nested instruction tag", func() { + config := configuration.NewConfiguration() + + for _, lines := range [][]string{ + {""}, + {""}, + {"ordinary text"}, + } { + context := newStateContext(lines...) + + err := parsing.EmbedInstruction.Accept(&context, config) + + var parseErr parsing.InstructionParseError + Expect(errors.As(err, &parseErr)).Should(BeTrue()) + Expect(parseErr.Reason).Should(ContainSubstring("opening `` tag is not closed")) + } + }) + It("should render source and close the embedding fence when the end state is accepted", func() { sourceRoot := GinkgoT().TempDir() Expect(os.WriteFile( diff --git a/embedding/processor.go b/embedding/processor.go index 9d8b4619..749e987e 100644 --- a/embedding/processor.go +++ b/embedding/processor.go @@ -73,10 +73,7 @@ func NewProcessor(docFile string, config configuration.Configuration) (Processor if err != nil { return Processor{}, err } - resolver, err := fragmentation.NewResolver(fragmentation.DefaultResolverCacheLimit) - if err != nil { - return Processor{}, err - } + resolver := newDefaultResolver() return newProcessor( docFile, @@ -87,6 +84,14 @@ func NewProcessor(docFile string, config configuration.Configuration) (Processor ), nil } +// newDefaultResolver creates a resolver with the fixed positive default cache limit. +// The constructor cannot fail because DefaultResolverCacheLimit satisfies its precondition. +func newDefaultResolver() *fragmentation.Resolver { + resolver, _ := fragmentation.NewResolver(fragmentation.DefaultResolverCacheLimit) + + return resolver +} + // newProcessor creates a Processor with a precomputed documentation file list. func newProcessor( docFile string,