diff --git a/.changeset/content-hash-cache-key.md b/.changeset/content-hash-cache-key.md new file mode 100644 index 00000000..0e84dc77 --- /dev/null +++ b/.changeset/content-hash-cache-key.md @@ -0,0 +1,6 @@ +--- +'@contentlayer2/source-files': patch +'@contentlayer2/core': patch +--- + +Key the document cache on a hash of the file contents instead of the file mtime. Git doesn't preserve mtimes, so on a fresh clone (CI, Vercel, etc.) every cached document was invalidated and re-rendered even when the persisted `.contentlayer/.cache` was restored. diff --git a/packages/@contentlayer/core/src/DataCache.ts b/packages/@contentlayer/core/src/DataCache.ts index 863959ec..38699d3d 100644 --- a/packages/@contentlayer/core/src/DataCache.ts +++ b/packages/@contentlayer/core/src/DataCache.ts @@ -27,7 +27,7 @@ export namespace DataCache { hasWarnings: boolean /** * The `documentHash` is used to determine if a document has changed and it's value-generation is implemented - * by a given plugin (e.g. based on the last-edit date in source-files) + * by a given plugin (e.g. a hash of the file contents in source-files) */ documentHash: string documentTypeName: string diff --git a/packages/@contentlayer/source-files/src/__test__/fetchData/makeCacheItemFromFilePath.spec.ts b/packages/@contentlayer/source-files/src/__test__/fetchData/makeCacheItemFromFilePath.spec.ts index 27302249..4fc0a85f 100644 --- a/packages/@contentlayer/source-files/src/__test__/fetchData/makeCacheItemFromFilePath.spec.ts +++ b/packages/@contentlayer/source-files/src/__test__/fetchData/makeCacheItemFromFilePath.spec.ts @@ -1,5 +1,7 @@ +import fs from 'node:fs' import path from 'node:path' +import type * as core from '@contentlayer2/core' import { E } from '@contentlayer2/utils/effect' import { fileURLToPath } from 'url' import { describe, expect, test } from 'vitest' @@ -98,6 +100,78 @@ describe('3-small-files', () => { }) }) +describe('3-small-files: cache', () => { + const contentDirPath = path.join(testFileDir, 'fixtures', '3-small-files') + + const TestPost = defineDocumentType(() => ({ + name: 'TestPost', + filePathPattern: `**/*.md`, + fields: {}, + })) + + const makeCacheItem = async (relativeFilePath: string): Promise => { + const { result } = await runTest({ documentTypes: [TestPost], contentDirPath, relativeFilePath }) + if (E.isLeft(result)) throw new Error(`Expected ${relativeFilePath} to parse: ${result.left._tag}`) + return result.right + } + + // Marks a cache item so a test can tell whether the result came from the cache or was re-parsed + const markAsCached = (cacheItem: core.DataCache.CacheItem): core.DataCache.CacheItem => ({ + ...cacheItem, + document: { ...cacheItem.document, fromCache: true }, + }) + + test('documentHash depends on file contents, not mtime', async () => { + const before = await makeCacheItem('a.md') + + // Simulate a fresh `git clone`: same contents, different mtime + const filePath = path.join(contentDirPath, 'a.md') + const oneHourLater = new Date(fs.statSync(filePath).mtimeMs + 60 * 60 * 1000) + fs.utimesSync(filePath, oneHourLater, oneHourLater) + + const after = await makeCacheItem('a.md') + const other = await makeCacheItem('b.md') + + expect(after.documentHash).toBe(before.documentHash) + expect(other.documentHash).not.toBe(before.documentHash) + }) + + test('returns the previous cache item when the hash matches', async () => { + const cacheItem = await makeCacheItem('a.md') + const previousCache: core.DataCache.Cache = { cacheItemsMap: { 'a.md': markAsCached(cacheItem) } } + + const { result } = await runTest({ + documentTypes: [TestPost], + contentDirPath, + relativeFilePath: 'a.md', + previousCache, + }) + + expect(result._tag).toBe('Right') + if (E.isRight(result)) { + expect(result.right.document.fromCache).toBe(true) + } + }) + + test('re-parses the document when the hash differs', async () => { + const staleCacheItem = { ...(await makeCacheItem('a.md')), documentHash: 'stale' } + const previousCache: core.DataCache.Cache = { cacheItemsMap: { 'a.md': markAsCached(staleCacheItem) } } + + const { result } = await runTest({ + documentTypes: [TestPost], + contentDirPath, + relativeFilePath: 'a.md', + previousCache, + }) + + expect(result._tag).toBe('Right') + if (E.isRight(result)) { + expect(result.right.document.fromCache).toBeUndefined() + expect(result.right.documentHash).not.toBe('stale') + } + }) +}) + describe('misc-files: empty-markdown', () => { const contentDirPath = path.join(testFileDir, 'fixtures', 'misc-files', 'empty-markdown') diff --git a/packages/@contentlayer/source-files/src/__test__/fetchData/utils.ts b/packages/@contentlayer/source-files/src/__test__/fetchData/utils.ts index fed9188f..380188be 100644 --- a/packages/@contentlayer/source-files/src/__test__/fetchData/utils.ts +++ b/packages/@contentlayer/source-files/src/__test__/fetchData/utils.ts @@ -22,10 +22,12 @@ export const runTest = async ({ documentTypes, contentDirPath: contentDirPath_, relativeFilePath: relativeFilePath_, + previousCache, }: { documentTypes: DocumentTypes contentDirPath: string relativeFilePath: string + previousCache?: core.DataCache.Cache }) => { const eff = T.gen(function* ($) { const relativeFilePath = unknownToRelativePosixFilePath(relativeFilePath_) @@ -59,7 +61,7 @@ export const runTest = async ({ coreSchemaDef, filePathPatternMap, options, - previousCache: undefined, + previousCache, contentTypeMap, }), These.effectToEither, diff --git a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts index 8190827d..f215ec3c 100644 --- a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts +++ b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts @@ -1,6 +1,6 @@ import type * as core from '@contentlayer2/core' import type { AbsolutePosixFilePath, RelativePosixFilePath } from '@contentlayer2/utils' -import { filePathJoin, fs } from '@contentlayer2/utils' +import { filePathJoin, fs, hashObject } from '@contentlayer2/utils' import type { HasConsole } from '@contentlayer2/utils/effect' import { identity, O, OT, pipe, T, These } from '@contentlayer2/utils/effect' import matter from 'gray-matter' @@ -40,12 +40,10 @@ export const makeCacheItemFromFilePath = ({ T.gen(function* ($) { const fullFilePath = filePathJoin(contentDirPath, relativeFilePath) - const documentHash = yield* $( - pipe( - fs.stat(fullFilePath), - T.map((_) => _.mtime.getTime().toString()), - ), - ) + // Hash the file contents (not the mtime): git doesn't preserve mtimes, so a fresh clone + // (e.g. CI, Vercel) would otherwise invalidate every cached document. + const fileContent = yield* $(fs.readFile(fullFilePath)) + const documentHash = yield* $(hashObject(fileContent)) // return previous cache item if it exists if ( @@ -60,7 +58,7 @@ export const makeCacheItemFromFilePath = ({ return These.succeed(cacheItem) } - const rawContent = yield* $(processRawContent({ fullFilePath, relativeFilePath })) + const rawContent = yield* $(processRawContent({ fileContent, relativeFilePath })) const [{ documentTypeDef }, warnings] = yield* $( pipe( @@ -111,9 +109,9 @@ export const makeCacheItemFromFilePath = ({ }), T.mapError((error) => { switch (error._tag) { - case 'fs.StatError': case 'fs.ReadFileError': case 'fs.FileNotFoundError': + case 'HashError': return new FetchDataError.UnexpectedError({ error, documentFilePath: relativeFilePath }) default: return error @@ -123,25 +121,22 @@ export const makeCacheItemFromFilePath = ({ ) const processRawContent = ({ - fullFilePath, + fileContent, relativeFilePath, }: { - fullFilePath: AbsolutePosixFilePath + fileContent: string relativeFilePath: RelativePosixFilePath }): T.Effect< - OT.HasTracer & fs.HasFs, + OT.HasTracer, | FetchDataError.UnsupportedFileExtension | FetchDataError.InvalidFrontmatterError | FetchDataError.InvalidMarkdownFileError | FetchDataError.InvalidJsonFileError - | FetchDataError.InvalidYamlFileError - | fs.FileNotFoundError - | fs.ReadFileError, + | FetchDataError.InvalidYamlFileError, RawContent > => pipe( T.gen(function* ($) { - const fileContent = yield* $(fs.readFile(fullFilePath)) const filePathExtension = relativeFilePath.toLowerCase().split('.').pop()! switch (filePathExtension) {