fix(source-files): key document cache on content hash, not mtime

.changeset/content-hash-cache-key.md

@@ -0,0 +1,6 @@
+---
+'@contentlayer2/source-files': patch
+'@contentlayer2/core': patch
+---
+
+Key the document cache on a hash of the file contents instead of the file mtime. Git doesn't preserve mtimes, so on a fresh clone (CI, Vercel, etc.) every cached document was invalidated and re-rendered even when the persisted `.contentlayer/.cache` was restored.

packages/@contentlayer/core/src/DataCache.ts

@@ -27,7 +27,7 @@ export namespace DataCache {
     hasWarnings: boolean
     /**
      * The `documentHash` is used to determine if a document has changed and it's value-generation is implemented
-     * by a given plugin (e.g. based on the last-edit date in source-files)
+     * by a given plugin (e.g. a hash of the file contents in source-files)
      */
     documentHash: string
     documentTypeName: string

packages/@contentlayer/source-files/src/__test__/fetchData/makeCacheItemFromFilePath.spec.ts

@@ -1,5 +1,7 @@
+import fs from 'node:fs'
 import path from 'node:path'
 
+import type * as core from '@contentlayer2/core'
 import { E } from '@contentlayer2/utils/effect'
 import { fileURLToPath } from 'url'
 import { describe, expect, test } from 'vitest'
@@ -98,6 +100,78 @@ describe('3-small-files', () => {
   })
 })
 
+describe('3-small-files: cache', () => {
+  const contentDirPath = path.join(testFileDir, 'fixtures', '3-small-files')
+
+  const TestPost = defineDocumentType(() => ({
+    name: 'TestPost',
+    filePathPattern: `**/*.md`,
+    fields: {},
+  }))
+
+  const makeCacheItem = async (relativeFilePath: string): Promise<core.DataCache.CacheItem> => {
+    const { result } = await runTest({ documentTypes: [TestPost], contentDirPath, relativeFilePath })
+    if (E.isLeft(result)) throw new Error(`Expected ${relativeFilePath} to parse: ${result.left._tag}`)
+    return result.right
+  }
+
+  // Marks a cache item so a test can tell whether the result came from the cache or was re-parsed
+  const markAsCached = (cacheItem: core.DataCache.CacheItem): core.DataCache.CacheItem => ({
+    ...cacheItem,
+    document: { ...cacheItem.document, fromCache: true },
+  })
+
+  test('documentHash depends on file contents, not mtime', async () => {
+    const before = await makeCacheItem('a.md')
+
+    // Simulate a fresh `git clone`: same contents, different mtime
+    const filePath = path.join(contentDirPath, 'a.md')
+    const oneHourLater = new Date(fs.statSync(filePath).mtimeMs + 60 * 60 * 1000)
+    fs.utimesSync(filePath, oneHourLater, oneHourLater)
+
+    const after = await makeCacheItem('a.md')
+    const other = await makeCacheItem('b.md')
+
+    expect(after.documentHash).toBe(before.documentHash)
+    expect(other.documentHash).not.toBe(before.documentHash)
+  })
+
+  test('returns the previous cache item when the hash matches', async () => {
+    const cacheItem = await makeCacheItem('a.md')
+    const previousCache: core.DataCache.Cache = { cacheItemsMap: { 'a.md': markAsCached(cacheItem) } }
+
+    const { result } = await runTest({
+      documentTypes: [TestPost],
+      contentDirPath,
+      relativeFilePath: 'a.md',
+      previousCache,
+    })
+
+    expect(result._tag).toBe('Right')
+    if (E.isRight(result)) {
+      expect(result.right.document.fromCache).toBe(true)
+    }
+  })
+
+  test('re-parses the document when the hash differs', async () => {
+    const staleCacheItem = { ...(await makeCacheItem('a.md')), documentHash: 'stale' }
+    const previousCache: core.DataCache.Cache = { cacheItemsMap: { 'a.md': markAsCached(staleCacheItem) } }
+
+    const { result } = await runTest({
+      documentTypes: [TestPost],
+      contentDirPath,
+      relativeFilePath: 'a.md',
+      previousCache,
+    })
+
+    expect(result._tag).toBe('Right')
+    if (E.isRight(result)) {
+      expect(result.right.document.fromCache).toBeUndefined()
+      expect(result.right.documentHash).not.toBe('stale')
+    }
+  })
+})
+
 describe('misc-files: empty-markdown', () => {
   const contentDirPath = path.join(testFileDir, 'fixtures', 'misc-files', 'empty-markdown')
 

packages/@contentlayer/source-files/src/__test__/fetchData/utils.ts

@@ -22,10 +22,12 @@ export const runTest = async ({
   documentTypes,
   contentDirPath: contentDirPath_,
   relativeFilePath: relativeFilePath_,
+  previousCache,
 }: {
   documentTypes: DocumentTypes
   contentDirPath: string
   relativeFilePath: string
+  previousCache?: core.DataCache.Cache
 }) => {
   const eff = T.gen(function* ($) {
     const relativeFilePath = unknownToRelativePosixFilePath(relativeFilePath_)
@@ -59,7 +61,7 @@ export const runTest = async ({
           coreSchemaDef,
           filePathPatternMap,
           options,
-          previousCache: undefined,
+          previousCache,
           contentTypeMap,
         }),
         These.effectToEither,

packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts

@@ -1,6 +1,6 @@
 import type * as core from '@contentlayer2/core'
 import type { AbsolutePosixFilePath, RelativePosixFilePath } from '@contentlayer2/utils'
-import { filePathJoin, fs } from '@contentlayer2/utils'
+import { filePathJoin, fs, hashObject } from '@contentlayer2/utils'
 import type { HasConsole } from '@contentlayer2/utils/effect'
 import { identity, O, OT, pipe, T, These } from '@contentlayer2/utils/effect'
 import matter from 'gray-matter'
@@ -40,12 +40,10 @@ export const makeCacheItemFromFilePath = ({
     T.gen(function* ($) {
       const fullFilePath = filePathJoin(contentDirPath, relativeFilePath)
 
-      const documentHash = yield* $(
-        pipe(
-          fs.stat(fullFilePath),
-          T.map((_) => _.mtime.getTime().toString()),
-        ),
-      )
+      // Hash the file contents (not the mtime): git doesn't preserve mtimes, so a fresh clone
+      // (e.g. CI, Vercel) would otherwise invalidate every cached document.
+      const fileContent = yield* $(fs.readFile(fullFilePath))
+      const documentHash = yield* $(hashObject(fileContent))
 
       // return previous cache item if it exists
       if (
@@ -60,7 +58,7 @@ export const makeCacheItemFromFilePath = ({
         return These.succeed(cacheItem)
       }
 
-      const rawContent = yield* $(processRawContent({ fullFilePath, relativeFilePath }))
+      const rawContent = yield* $(processRawContent({ fileContent, relativeFilePath }))
 
       const [{ documentTypeDef }, warnings] = yield* $(
         pipe(
@@ -111,9 +109,9 @@ export const makeCacheItemFromFilePath = ({
     }),
     T.mapError((error) => {
       switch (error._tag) {
-        case 'fs.StatError':
         case 'fs.ReadFileError':
         case 'fs.FileNotFoundError':
+        case 'HashError':
           return new FetchDataError.UnexpectedError({ error, documentFilePath: relativeFilePath })
         default:
           return error
@@ -123,25 +121,22 @@ export const makeCacheItemFromFilePath = ({
   )
 
 const processRawContent = ({
-  fullFilePath,
+  fileContent,
   relativeFilePath,
 }: {
-  fullFilePath: AbsolutePosixFilePath
+  fileContent: string
   relativeFilePath: RelativePosixFilePath
 }): T.Effect<
-  OT.HasTracer & fs.HasFs,
+  OT.HasTracer,
   | FetchDataError.UnsupportedFileExtension
   | FetchDataError.InvalidFrontmatterError
   | FetchDataError.InvalidMarkdownFileError
   | FetchDataError.InvalidJsonFileError
-  | FetchDataError.InvalidYamlFileError
-  | fs.FileNotFoundError
-  | fs.ReadFileError,
+  | FetchDataError.InvalidYamlFileError,
   RawContent
 > =>
   pipe(
     T.gen(function* ($) {
-      const fileContent = yield* $(fs.readFile(fullFilePath))
       const filePathExtension = relativeFilePath.toLowerCase().split('.').pop()!
 
       switch (filePathExtension) {