From 968e0bdd5c48f33cbf7700d380793fcdc01c341d Mon Sep 17 00:00:00 2001 From: Nate Lentz Date: Mon, 27 Jul 2026 14:22:42 -0400 Subject: [PATCH] fix(plugin-import-export): import fields whose names contain underscores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `unflattenObject` rebuilt the nested document by splitting every CSV column name on `_`, so a field whose own name contains an underscore was read as a nested path: `vat_number` became `{ vat: { number } }`. The phantom `vat` key is not a real field, so the collection operation stripped it and the column imported as a no-op. Resolve column names against the field schema instead. `flatKeyToPathSegments` matches field names longest-first with backtracking, emitting array indices, block slugs and locale codes as their own segments — so a key the naive split already handled correctly resolves to the identical segments. Keys the schema cannot account for (extra columns, the populated relationship data an export at depth > 0 produces) return `undefined` and fall back to the naive split, leaving their handling unchanged. Groups, named tabs, arrays, blocks and localized fields are all resolved recursively, including underscores in the container's own name. Where a flat field and a group path collide (`vat_number` vs. a `vat` group containing `number`) the flat field wins; that schema is ambiguous on export too, since both produce a single `vat_number` column. Fixes #17118 --- .../utilities/flatKeyToPathSegments.spec.ts | 400 ++++++++++++++++++ .../src/utilities/flatKeyToPathSegments.ts | 306 ++++++++++++++ .../src/utilities/unflattenObject.spec.ts | 159 +++++++ .../src/utilities/unflattenObject.ts | 14 +- .../collections/PostsWithSnakeCaseFields.ts | 49 +++ test/plugin-import-export/config.ts | 2 + test/plugin-import-export/int.spec.ts | 183 +++++++- test/plugin-import-export/payload-types.ts | 233 +++++++++- test/plugin-import-export/shared.ts | 2 + 9 files changed, 1326 insertions(+), 22 deletions(-) create mode 100644 packages/plugin-import-export/src/utilities/flatKeyToPathSegments.spec.ts create mode 100644 packages/plugin-import-export/src/utilities/flatKeyToPathSegments.ts create mode 100644 test/plugin-import-export/collections/PostsWithSnakeCaseFields.ts diff --git a/packages/plugin-import-export/src/utilities/flatKeyToPathSegments.spec.ts b/packages/plugin-import-export/src/utilities/flatKeyToPathSegments.spec.ts new file mode 100644 index 00000000000..a53c2a0553d --- /dev/null +++ b/packages/plugin-import-export/src/utilities/flatKeyToPathSegments.spec.ts @@ -0,0 +1,400 @@ +import type { FlattenedBlock, FlattenedField } from 'payload' + +import { flatKeyToPathSegments } from './flatKeyToPathSegments.js' + +import { describe, it, expect } from 'vitest' + +describe('flatKeyToPathSegments', () => { + describe('flat field names containing underscores', () => { + const fields: FlattenedField[] = [ + { name: 'vat_number', type: 'text' } as FlattenedField, + { name: '_start_with_underscore', type: 'text' } as FlattenedField, + { name: 'with_numbers_1', type: 'text' } as FlattenedField, + ] + + it('should keep a snake_case name as a single segment', () => { + expect(flatKeyToPathSegments({ fields, flatKey: 'vat_number' })).toEqual(['vat_number']) + }) + + it('should keep a leading underscore', () => { + expect(flatKeyToPathSegments({ fields, flatKey: '_start_with_underscore' })).toEqual([ + '_start_with_underscore', + ]) + }) + + it('should not treat a trailing digit in a name as an array index', () => { + expect(flatKeyToPathSegments({ fields, flatKey: 'with_numbers_1' })).toEqual([ + 'with_numbers_1', + ]) + }) + }) + + describe('groups', () => { + it('should resolve a snake_case field nested in a group', () => { + const fields: FlattenedField[] = [ + { + name: 'group', + type: 'group', + flattenedFields: [{ name: 'vat_number', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'group_vat_number' })).toEqual([ + 'group', + 'vat_number', + ]) + }) + + it('should resolve a group whose own name contains underscores', () => { + const fields: FlattenedField[] = [ + { + name: 'billing_details', + type: 'group', + flattenedFields: [{ name: 'vat_number', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'billing_details_vat_number' })).toEqual([ + 'billing_details', + 'vat_number', + ]) + }) + + it('should prefer a flat field over descending into a group with a colliding name', () => { + const fields: FlattenedField[] = [ + { name: 'vat_number', type: 'text' } as FlattenedField, + { + name: 'vat', + type: 'group', + flattenedFields: [{ name: 'number', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'vat_number' })).toEqual(['vat_number']) + }) + + it('should backtrack to a group when the longer flat name cannot resolve the whole key', () => { + const fields: FlattenedField[] = [ + { name: 'vat_number', type: 'text' } as FlattenedField, + { + name: 'vat', + type: 'group', + flattenedFields: [{ name: 'number_suffix', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'vat_number_suffix' })).toEqual([ + 'vat', + 'number_suffix', + ]) + }) + + it('should resolve nested groups that both contain underscores', () => { + const fields: FlattenedField[] = [ + { + name: 'outer_group', + type: 'group', + flattenedFields: [ + { + name: 'inner_group', + type: 'group', + flattenedFields: [{ name: 'my_field', type: 'text' }], + }, + ], + } as unknown as FlattenedField, + ] + + expect( + flatKeyToPathSegments({ fields, flatKey: 'outer_group_inner_group_my_field' }), + ).toEqual(['outer_group', 'inner_group', 'my_field']) + }) + + it('should resolve a named tab like a group', () => { + const fields: FlattenedField[] = [ + { + name: 'meta_tab', + type: 'tab', + flattenedFields: [{ name: 'vat_number', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'meta_tab_vat_number' })).toEqual([ + 'meta_tab', + 'vat_number', + ]) + }) + + it('should not resolve a group that carries no flattenedFields', () => { + const fields: FlattenedField[] = [{ name: 'group', type: 'group' } as FlattenedField] + + expect(flatKeyToPathSegments({ fields, flatKey: 'group_field1' })).toBeUndefined() + }) + }) + + describe('localized fields', () => { + const localeCodes = ['en', 'en_US', 'es'] + + it('should emit the locale as its own segment', () => { + const fields: FlattenedField[] = [ + { name: 'vat_number', type: 'text', localized: true } as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'vat_number_en', localeCodes })).toEqual([ + 'vat_number', + 'en', + ]) + }) + + it('should prefer a longer locale code over a shorter one', () => { + const fields: FlattenedField[] = [ + { name: 'title', type: 'text', localized: true } as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'title_en_US', localeCodes })).toEqual([ + 'title', + 'en_US', + ]) + }) + + it('should recognize a locale by shape when no locale codes are configured', () => { + const fields: FlattenedField[] = [ + { name: 'vat_number', type: 'text', localized: true } as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'vat_number_es' })).toEqual([ + 'vat_number', + 'es', + ]) + }) + + it('should resolve a field inside a localized group', () => { + const fields: FlattenedField[] = [ + { + name: 'group', + type: 'group', + localized: true, + flattenedFields: [{ name: 'vat_number', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect( + flatKeyToPathSegments({ fields, flatKey: 'group_en_vat_number', localeCodes }), + ).toEqual(['group', 'en', 'vat_number']) + }) + + it('should backtrack when a locale-shaped segment is actually part of a field name', () => { + const fields: FlattenedField[] = [ + { + name: 'group', + type: 'group', + localized: true, + flattenedFields: [{ name: 'es_label', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'group_es_label' })).toEqual([ + 'group', + 'es_label', + ]) + }) + + it('should not consume a locale suffix on a field that is not localized', () => { + const fields: FlattenedField[] = [{ name: 'title', type: 'text' } as FlattenedField] + + expect(flatKeyToPathSegments({ fields, flatKey: 'title_en', localeCodes })).toBeUndefined() + }) + }) + + describe('arrays and hasMany fields', () => { + it('should resolve a snake_case field inside an array row', () => { + const fields: FlattenedField[] = [ + { + name: 'line_items', + type: 'array', + flattenedFields: [{ name: 'vat_number', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'line_items_0_vat_number' })).toEqual([ + 'line_items', + '0', + 'vat_number', + ]) + }) + + it('should resolve an indexed hasMany scalar column', () => { + const fields: FlattenedField[] = [ + { name: 'my_tags', type: 'text', hasMany: true } as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'my_tags_0' })).toEqual(['my_tags', '0']) + }) + + it('should not treat an index suffix as valid on a field that is not hasMany', () => { + const fields: FlattenedField[] = [{ name: 'my_tags', type: 'text' } as FlattenedField] + + expect(flatKeyToPathSegments({ fields, flatKey: 'my_tags_0' })).toBeUndefined() + }) + }) + + describe('blocks', () => { + const fields: FlattenedField[] = [ + { + name: 'page_content', + type: 'blocks', + blocks: [ + { + slug: 'hero_section', + flattenedFields: [{ name: 'call_to_action', type: 'text' }], + }, + ], + } as unknown as FlattenedField, + ] + + it('should resolve an underscored block slug and field name', () => { + expect( + flatKeyToPathSegments({ fields, flatKey: 'page_content_0_hero_section_call_to_action' }), + ).toEqual(['page_content', '0', 'hero_section', 'call_to_action']) + }) + + it('should resolve the blockType column', () => { + expect( + flatKeyToPathSegments({ fields, flatKey: 'page_content_0_hero_section_blockType' }), + ).toEqual(['page_content', '0', 'hero_section', 'blockType']) + }) + + it('should resolve a block referenced by slug', () => { + const referencingFields: FlattenedField[] = [ + { + name: 'page_content', + type: 'blocks', + blocks: ['hero_section'], + } as unknown as FlattenedField, + ] + + const blocksBySlug: Record = { + hero_section: { + slug: 'hero_section', + fields: [], + flattenedFields: [{ name: 'call_to_action', type: 'text' }], + } as unknown as FlattenedBlock, + } + + expect( + flatKeyToPathSegments({ + blocksBySlug, + fields: referencingFields, + flatKey: 'page_content_0_hero_section_call_to_action', + }), + ).toEqual(['page_content', '0', 'hero_section', 'call_to_action']) + }) + + it('should not resolve a block reference that cannot be looked up', () => { + const referencingFields: FlattenedField[] = [ + { + name: 'page_content', + type: 'blocks', + blocks: ['hero_section'], + } as unknown as FlattenedField, + ] + + expect( + flatKeyToPathSegments({ + fields: referencingFields, + flatKey: 'page_content_0_hero_section_call_to_action', + }), + ).toBeUndefined() + }) + }) + + describe('relationships', () => { + it('should resolve the id and relationTo columns of a polymorphic relationship', () => { + const fields: FlattenedField[] = [ + { + name: 'related_doc', + type: 'relationship', + relationTo: ['posts', 'pages'], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'related_doc_id' })).toEqual([ + 'related_doc', + 'id', + ]) + expect(flatKeyToPathSegments({ fields, flatKey: 'related_doc_relationTo' })).toEqual([ + 'related_doc', + 'relationTo', + ]) + }) + + it('should resolve indexed columns of a polymorphic hasMany relationship', () => { + const fields: FlattenedField[] = [ + { + name: 'related_docs', + type: 'relationship', + hasMany: true, + relationTo: ['posts', 'pages'], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'related_docs_0_relationTo' })).toEqual([ + 'related_docs', + '0', + 'relationTo', + ]) + }) + + it('should not resolve populated relationship data', () => { + const fields: FlattenedField[] = [ + { name: 'author', type: 'relationship', relationTo: 'users' } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'author_email_address' })).toBeUndefined() + }) + }) + + describe('unresolvable keys', () => { + const fields: FlattenedField[] = [{ name: 'title', type: 'text' } as FlattenedField] + + it('should not resolve a column that is absent from the schema', () => { + expect(flatKeyToPathSegments({ fields, flatKey: 'my_extra_column' })).toBeUndefined() + }) + + it('should not resolve a partial field path', () => { + const groupFields: FlattenedField[] = [ + { + name: 'group', + type: 'group', + flattenedFields: [{ name: 'nested', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields: groupFields, flatKey: 'group' })).toBeUndefined() + }) + }) + + it('should match a naive underscore split when no field name contains an underscore', () => { + const fields: FlattenedField[] = [ + { name: 'title', type: 'text', localized: true } as FlattenedField, + { + name: 'items', + type: 'array', + flattenedFields: [{ name: 'name', type: 'text' }], + } as unknown as FlattenedField, + { + name: 'group', + type: 'group', + flattenedFields: [{ name: 'nested', type: 'text' }], + } as unknown as FlattenedField, + { name: 'poly', type: 'relationship', relationTo: ['posts'] } as unknown as FlattenedField, + ] + + const keys = ['title_en', 'items_0_name', 'group_nested', 'poly_id', 'poly_relationTo'] + + for (const flatKey of keys) { + expect(flatKeyToPathSegments({ fields, flatKey, localeCodes: ['en'] })).toEqual( + flatKey.split('_'), + ) + } + }) +}) diff --git a/packages/plugin-import-export/src/utilities/flatKeyToPathSegments.ts b/packages/plugin-import-export/src/utilities/flatKeyToPathSegments.ts new file mode 100644 index 00000000000..3fdd967d64e --- /dev/null +++ b/packages/plugin-import-export/src/utilities/flatKeyToPathSegments.ts @@ -0,0 +1,306 @@ +import type { FlattenedBlock, FlattenedField } from 'payload' + +import { getBlockFlattenedFields, getNestedFlattenedFields } from './flattenedFields.js' + +type Args = { + /** + * Resolved block definitions, used to look up blocks that a `blocks` field + * references by slug rather than defines inline. Pass `req.payload.blocks`. + */ + blocksBySlug?: Record + fields: FlattenedField[] + flatKey: string + /** + * Configured locale codes. When omitted, a locale segment is recognized by + * shape (`en`, `en_US`) instead — the same heuristic `unflattenPostProcess` uses. + */ + localeCodes?: string[] +} + +type ResolveContext = Pick + +/** `null` rest means the prefix consumed the whole key. */ +type Taken = { + rest: null | string +} + +const indexSegment = /^\d+$/ + +const localeSegment = /^[a-z]{2}(?:_[A-Z]{2})?$/ + +const segmentsCache = new WeakMap>() + +/** + * Resolves a flattened CSV column name into the path segments that address it + * within a document, using the field schema rather than assuming every + * underscore is a nesting separator. `vat_number` resolves to `['vat_number']`, + * while `group_vat_number` resolves to `['group', 'vat_number']`. + * + * Array indices, block slugs and locale codes are emitted as their own segments, + * so a key the naive `split('_')` already handled correctly resolves to the + * identical segments. + * + * Returns `undefined` when the schema cannot account for the key — extra CSV + * columns, or the populated relationship data an export at depth > 0 produces. + * Callers are expected to fall back to `flatKey.split('_')` in that case. + * + * Field names are matched longest-first with backtracking, so a flat field named + * `vat_number` wins over a `vat` group containing `number`. Such a schema is + * ambiguous on export too — both produce a single `vat_number` column — so the + * collision cannot be resolved from the CSV alone. + */ +export const flatKeyToPathSegments = ({ + blocksBySlug, + fields, + flatKey, + localeCodes, +}: Args): string[] | undefined => { + let cached = segmentsCache.get(fields) + + if (!cached) { + cached = new Map() + segmentsCache.set(fields, cached) + } + + // Resolution depends on the locales as well as the schema, and a column set is + // re-resolved for every imported row — worth memoizing across rows. + const cacheKey = `${localeCodes?.join(',') ?? ''}:${flatKey}` + const memoized = cached.get(cacheKey) + + if (memoized !== undefined) { + return memoized ?? undefined + } + + const resolved = resolveInFields(fields, flatKey, { blocksBySlug, localeCodes }) + cached.set(cacheKey, resolved ?? null) + + return resolved +} + +const resolveInFields = ( + fields: FlattenedField[], + key: string, + ctx: ResolveContext, +): string[] | undefined => { + const candidates = fields + .filter((field): field is { name: string } & FlattenedField => + Boolean('name' in field && field.name), + ) + .sort((a, b) => b.name.length - a.name.length) + + for (const field of candidates) { + const taken = takePrefix(key, field.name) + if (!taken) { + continue + } + + const suffix = resolveField({ ctx, field, tail: taken.rest }) + if (suffix) { + return [field.name, ...suffix] + } + } + + return undefined +} + +/** + * Consumes an optional locale segment before the field's own suffix handling, so + * both `title_en` and localized-group keys like `group_en_vat_number` resolve. + * Falls back to treating the tail as part of the field when the locale reading + * doesn't resolve. + */ +const resolveField = ({ + ctx, + field, + tail, +}: { + ctx: ResolveContext + field: FlattenedField + tail: null | string +}): string[] | undefined => { + if (field.localized && tail !== null) { + const taken = takeLocale(tail, ctx.localeCodes) + + if (taken) { + const suffix = resolveFieldBody({ ctx, field, tail: taken.rest }) + if (suffix) { + return [taken.locale, ...suffix] + } + } + } + + return resolveFieldBody({ ctx, field, tail }) +} + +const resolveFieldBody = ({ + ctx, + field, + tail, +}: { + ctx: ResolveContext + field: FlattenedField + tail: null | string +}): string[] | undefined => { + switch (field.type) { + case 'array': { + if (tail === null) { + return undefined + } + + const [index, rest] = splitFirstSegment(tail) + if (!indexSegment.test(index)) { + return undefined + } + if (rest === null) { + return [index] + } + + const suffix = resolveInFields(getNestedFlattenedFields(field) ?? [], rest, ctx) + + return suffix && [index, ...suffix] + } + + case 'blocks': { + if (tail === null) { + return undefined + } + + const [index, rest] = splitFirstSegment(tail) + if (!indexSegment.test(index) || rest === null) { + return undefined + } + + return resolveInBlocks({ ctx, field, index, key: rest }) + } + + case 'group': + case 'tab': { + if (tail === null) { + return undefined + } + + return resolveInFields(getNestedFlattenedFields(field) ?? [], tail, ctx) + } + + case 'relationship': + case 'upload': { + if (tail === null) { + return [] + } + + const [index, rest] = splitFirstSegment(tail) + + // hasMany: `rel_0`, or `rel_0_id` / `rel_0_relationTo` when polymorphic + if (indexSegment.test(index)) { + if (rest === null) { + return [index] + } + + return isPolymorphicSuffix(rest) ? [index, rest] : undefined + } + + // hasOne polymorphic: `rel_id` / `rel_relationTo` + return isPolymorphicSuffix(tail) ? [tail] : undefined + } + + default: { + if (tail === null) { + return [] + } + + // hasMany scalars are exported one column per index, e.g. `tags_0` + const isHasMany = 'hasMany' in field && field.hasMany === true + + return isHasMany && indexSegment.test(tail) ? [tail] : undefined + } + } +} + +const resolveInBlocks = ({ + ctx, + field, + index, + key, +}: { + ctx: ResolveContext + field: FlattenedField + index: string + key: string +}): string[] | undefined => { + const blocks = 'blocks' in field ? (field.blocks ?? []) : [] + + const definitions = blocks + .map((block) => (typeof block === 'string' ? ctx.blocksBySlug?.[block] : block)) + .filter((block): block is FlattenedBlock => Boolean(block)) + .sort((a, b) => b.slug.length - a.slug.length) + + for (const block of definitions) { + const taken = takePrefix(key, block.slug) + if (!taken || taken.rest === null) { + continue + } + + if (taken.rest === 'blockType') { + return [index, block.slug, 'blockType'] + } + + const suffix = resolveInFields(getBlockFlattenedFields(block), taken.rest, ctx) + if (suffix) { + return [index, block.slug, ...suffix] + } + } + + return undefined +} + +const takePrefix = (key: string, prefix: string): Taken | undefined => { + if (key === prefix) { + return { rest: null } + } + + if (key.startsWith(`${prefix}_`)) { + return { rest: key.slice(prefix.length + 1) } + } + + return undefined +} + +const takeLocale = ( + key: string, + localeCodes: string[] | undefined, +): ({ locale: string } & Taken) | undefined => { + for (const candidate of localeCandidates(key, localeCodes)) { + const taken = takePrefix(key, candidate) + if (taken) { + return { locale: candidate, rest: taken.rest } + } + } + + return undefined +} + +/** + * Locale codes to try against the head of `key`, longest first so `en_US` wins + * over a co-existing `en`. Without configured codes, the first one or two + * segments are offered when they look like a locale. + */ +const localeCandidates = (key: string, localeCodes: string[] | undefined): string[] => { + if (localeCodes && localeCodes.length > 0) { + return [...localeCodes].sort((a, b) => b.length - a.length) + } + + const [first, second] = key.split('_') + + return [second === undefined ? undefined : `${first}_${second}`, first].filter( + (candidate): candidate is string => candidate !== undefined && localeSegment.test(candidate), + ) +} + +const splitFirstSegment = (key: string): [string, null | string] => { + const boundary = key.indexOf('_') + + return boundary === -1 ? [key, null] : [key.slice(0, boundary), key.slice(boundary + 1)] +} + +const isPolymorphicSuffix = (segment: string): boolean => + segment === 'id' || segment === 'relationTo' diff --git a/packages/plugin-import-export/src/utilities/unflattenObject.spec.ts b/packages/plugin-import-export/src/utilities/unflattenObject.spec.ts index 681f1f35359..70d39b30b8b 100644 --- a/packages/plugin-import-export/src/utilities/unflattenObject.spec.ts +++ b/packages/plugin-import-export/src/utilities/unflattenObject.spec.ts @@ -367,6 +367,165 @@ describe('unflattenObject', () => { }) }) + describe('field names containing underscores', () => { + const fields: FlattenedField[] = [ + { + name: 'simple', + type: 'text', + } as FlattenedField, + { + name: 'camelCase', + type: 'text', + } as FlattenedField, + { + name: 'PascalCase', + type: 'text', + } as FlattenedField, + { + name: 'with_underscores', + type: 'text', + } as FlattenedField, + { + name: '_start_with_underscore', + type: 'text', + } as FlattenedField, + { + name: 'with_numbers_1', + type: 'text', + } as FlattenedField, + { + name: 'localized_with_underscores_with_numbers_1', + type: 'text', + localized: true, + } as FlattenedField, + ] + + it('should keep a snake_case field flat instead of nesting it', () => { + const data = { + vat_number: 'IT12345678901', + } + + const result = unflattenObject({ + data, + fields: [{ name: 'vat_number', type: 'text' } as FlattenedField], + req: mockReq, + }) + + expect(result).toEqual({ + vat_number: 'IT12345678901', + }) + }) + + it('should handle all permitted field name shapes at the top level', () => { + const data = { + simple: 'simple', + camelCase: 'camelCase', + PascalCase: 'PascalCase', + with_underscores: 'with_underscores', + _start_with_underscore: '_start_with_underscore', + with_numbers_1: 'with_numbers_1', + localized_with_underscores_with_numbers_1_en: + 'localized_with_underscores_with_numbers_1_en', + localized_with_underscores_with_numbers_1_es: + 'localized_with_underscores_with_numbers_1_es', + } + + const result = unflattenObject({ data, fields, req: mockReq }) + + expect(result).toEqual({ + simple: 'simple', + camelCase: 'camelCase', + PascalCase: 'PascalCase', + with_underscores: 'with_underscores', + _start_with_underscore: '_start_with_underscore', + with_numbers_1: 'with_numbers_1', + localized_with_underscores_with_numbers_1: { + en: 'localized_with_underscores_with_numbers_1_en', + es: 'localized_with_underscores_with_numbers_1_es', + }, + }) + }) + + it('should handle all permitted field name shapes nested in a group', () => { + const groupFields: FlattenedField[] = [ + { + name: 'group', + type: 'group', + flattenedFields: fields, + } as unknown as FlattenedField, + ] + + const data = { + group_simple: 'simple', + group_camelCase: 'camelCase', + group_PascalCase: 'PascalCase', + group_with_underscores: 'with_underscores', + group__start_with_underscore: '_start_with_underscore', + group_with_numbers_1: 'with_numbers_1', + group_localized_with_underscores_with_numbers_1_en: + 'localized_with_underscores_with_numbers_1_en', + group_localized_with_underscores_with_numbers_1_es: + 'localized_with_underscores_with_numbers_1_es', + } + + const result = unflattenObject({ data, fields: groupFields, req: mockReq }) + + expect(result).toEqual({ + group: { + simple: 'simple', + camelCase: 'camelCase', + PascalCase: 'PascalCase', + with_underscores: 'with_underscores', + _start_with_underscore: '_start_with_underscore', + with_numbers_1: 'with_numbers_1', + localized_with_underscores_with_numbers_1: { + en: 'localized_with_underscores_with_numbers_1_en', + es: 'localized_with_underscores_with_numbers_1_es', + }, + }, + }) + }) + + it('should handle a snake_case field inside an array row', () => { + const arrayFields: FlattenedField[] = [ + { + name: 'items', + type: 'array', + flattenedFields: [{ name: 'vat_number', type: 'text' }], + } as unknown as FlattenedField, + ] + + const data = { + items_0_vat_number: 'IT12345678901', + } + + const result = unflattenObject({ data, fields: arrayFields, req: mockReq }) + + expect(result).toEqual({ + items: [{ vat_number: 'IT12345678901' }], + }) + }) + + it('should not treat a text field named with an _id suffix as a relationship', () => { + const idSuffixFields: FlattenedField[] = [ + { + name: 'external_id', + type: 'text', + } as FlattenedField, + ] + + const data = { + external_id: 'abc-123', + } + + const result = unflattenObject({ data, fields: idSuffixFields, req: mockReq }) + + expect(result).toEqual({ + external_id: 'abc-123', + }) + }) + }) + describe('polymorphic relationships', () => { const fields: FlattenedField[] = [ { diff --git a/packages/plugin-import-export/src/utilities/unflattenObject.ts b/packages/plugin-import-export/src/utilities/unflattenObject.ts index a079c51b18d..a127e8c3346 100644 --- a/packages/plugin-import-export/src/utilities/unflattenObject.ts +++ b/packages/plugin-import-export/src/utilities/unflattenObject.ts @@ -2,6 +2,7 @@ import type { FlattenedField, PayloadRequest } from 'payload' import type { ImportFieldHookEntry } from '../types.js' +import { flatKeyToPathSegments } from './flatKeyToPathSegments.js' import { getBlockFlattenedFields, getNestedFlattenedFields } from './flattenedFields.js' import { postProcessDocument } from './unflattenPostProcess.js' @@ -79,6 +80,14 @@ export const unflattenObject = ({ const arrayLikeNames = new Set() collectArrayLikeNames(fields, arrayLikeNames) + const localization = req?.payload?.config?.localization + + const segmentArgs = { + blocksBySlug: req?.payload?.blocks, + fields, + localeCodes: localization ? localization.localeCodes : undefined, + } + // Sort keys to ensure array indices are processed in order const sortedKeys = Object.keys(data).sort((a, b) => { // Extract array indices from flattened keys (e.g., "field_0_subfield" -> "0") @@ -184,7 +193,10 @@ export const unflattenObject = ({ } // Example: "blocks_0_content_text" -> ["blocks", "0", "content", "text"] - const pathSegments = flatKey.split('_') + // Resolved against the field schema so names that contain underscores stay + // intact ("vat_number"), falling back to a naive split for keys the schema + // cannot account for — extra CSV columns, or populated relationship data. + const pathSegments = flatKeyToPathSegments({ ...segmentArgs, flatKey }) ?? flatKey.split('_') let currentObject: Record = result for (let i = 0; i < pathSegments.length; i++) { diff --git a/test/plugin-import-export/collections/PostsWithSnakeCaseFields.ts b/test/plugin-import-export/collections/PostsWithSnakeCaseFields.ts new file mode 100644 index 00000000000..8226a0ee292 --- /dev/null +++ b/test/plugin-import-export/collections/PostsWithSnakeCaseFields.ts @@ -0,0 +1,49 @@ +import type { CollectionConfig } from 'payload' + +import { postsWithSnakeCaseFieldsSlug } from '../shared.js' + +export const PostsWithSnakeCaseFields: CollectionConfig = { + slug: postsWithSnakeCaseFieldsSlug, + labels: { + singular: 'Posts With Snake Case Fields', + plural: 'Posts With Snake Case Fields', + }, + admin: { + useAsTitle: 'title', + }, + fields: [ + { + name: 'title', + type: 'text', + }, + { + name: 'vat_number', + type: 'text', + }, + { + name: 'billing_details', + type: 'group', + fields: [ + { + name: 'vat_number', + type: 'text', + }, + ], + }, + { + name: 'line_items', + type: 'array', + fields: [ + { + name: 'item_code', + type: 'text', + }, + ], + }, + { + name: 'localized_note', + type: 'text', + localized: true, + }, + ], +} diff --git a/test/plugin-import-export/config.ts b/test/plugin-import-export/config.ts index 86dc1c4080e..381328d3e06 100644 --- a/test/plugin-import-export/config.ts +++ b/test/plugin-import-export/config.ts @@ -26,6 +26,7 @@ import { PostsWithFieldHooks } from './collections/PostsWithFieldHooks.js' import { PostsWithHooks } from './collections/PostsWithHooks.js' import { PostsWithLimits } from './collections/PostsWithLimits.js' import { PostsWithS3 } from './collections/PostsWithS3.js' +import { PostsWithSnakeCaseFields } from './collections/PostsWithSnakeCaseFields.js' import { Users } from './collections/Users.js' import { exportAfterHook, @@ -71,6 +72,7 @@ export default buildConfigWithDefaults({ PostsWithHooks, PostsWithFieldHooks, PostsWithColumnMap, + PostsWithSnakeCaseFields, Media, CustomIdPages, ], diff --git a/test/plugin-import-export/int.spec.ts b/test/plugin-import-export/int.spec.ts index 97ab2f1348d..09e121dfa81 100644 --- a/test/plugin-import-export/int.spec.ts +++ b/test/plugin-import-export/int.spec.ts @@ -14,7 +14,7 @@ import { devUser, regularUser } from '../credentials.js' import { clearTestBucket, createTestBucket } from '../storage-s3/test-utils.js' import { readCSV, readJSON } from './helpers.js' import { richTextData } from './seed/richTextData.js' -import { customIdPagesSlug, postsWithS3Slug } from './shared.js' +import { customIdPagesSlug, postsWithS3Slug, postsWithSnakeCaseFieldsSlug } from './shared.js' let payload: Payload let restClient: NextRESTClient @@ -8907,4 +8907,185 @@ describe('@payloadcms/plugin-import-export', () => { posts.docs.forEach((post) => createdPostIDs.push(post.id)) }) }) + + describe('field names containing underscores', () => { + const createdPostIDs: (number | string)[] = [] + + afterEach(async () => { + for (const id of createdPostIDs) { + await payload.delete({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + id, + }) + } + createdPostIDs.length = 0 + }) + + const importCSV = async (csvContent: string) => { + const csvBuffer = Buffer.from(csvContent) + + const importDoc = await payload.create({ + collection: 'imports', + user, + data: { + collectionSlug: postsWithSnakeCaseFieldsSlug, + importMode: 'update', + matchField: 'id', + }, + file: { + data: csvBuffer, + mimetype: 'text/csv', + name: 'snake-case-import-test.csv', + size: csvBuffer.length, + }, + }) + + await payload.jobs.run() + + return payload.findByID({ + collection: 'imports', + id: importDoc.id, + }) + } + + it('should update a snake_case field from CSV', async () => { + const post = await payload.create({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + data: { + title: 'Snake Case Import Test', + vat_number: 'IT00000000000', + }, + }) + + createdPostIDs.push(post.id) + + const importDoc = await importCSV( + `id,title,vat_number\n${post.id},"Snake Case Import Test","IT12345678901"`, + ) + + expect(importDoc.status).toBe('completed') + expect(importDoc.summary?.updated).toBe(1) + expect(importDoc.summary?.issues).toBe(0) + + const updated = await payload.findByID({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + id: post.id, + }) + + expect(updated.vat_number).toBe('IT12345678901') + }) + + it('should update snake_case fields nested in a group and an array from CSV', async () => { + const post = await payload.create({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + data: { + title: 'Snake Case Nested Test', + billing_details: { vat_number: 'IT00000000000' }, + line_items: [{ item_code: 'OLD-1' }], + }, + }) + + createdPostIDs.push(post.id) + + const importDoc = await importCSV( + `id,billing_details_vat_number,line_items_0_item_code\n${post.id},"IT12345678901","NEW-1"`, + ) + + expect(importDoc.status).toBe('completed') + expect(importDoc.summary?.issues).toBe(0) + + const updated = await payload.findByID({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + id: post.id, + }) + + expect(updated.billing_details?.vat_number).toBe('IT12345678901') + expect(updated.line_items?.[0]?.item_code).toBe('NEW-1') + }) + + it('should update a localized snake_case field per locale from CSV', async () => { + const post = await payload.create({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + data: { + title: 'Snake Case Localized Test', + localized_note: 'english original', + }, + }) + + createdPostIDs.push(post.id) + + const importDoc = await importCSV( + `id,localized_note_en,localized_note_es\n${post.id},"english updated","spanish updated"`, + ) + + expect(importDoc.status).toBe('completed') + expect(importDoc.summary?.issues).toBe(0) + + const english = await payload.findByID({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + id: post.id, + locale: 'en', + }) + const spanish = await payload.findByID({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + id: post.id, + locale: 'es', + }) + + expect(english.localized_note).toBe('english updated') + expect(spanish.localized_note).toBe('spanish updated') + }) + + it('should round-trip snake_case fields through export and import', async () => { + const post = await payload.create({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + data: { + title: 'Snake Case Round Trip', + vat_number: 'IT12345678901', + billing_details: { vat_number: 'IT99999999999' }, + }, + }) + + createdPostIDs.push(post.id) + + const exportDoc = await payload.create({ + collection: 'exports', + user, + data: { + collectionSlug: postsWithSnakeCaseFieldsSlug, + fields: ['id', 'vat_number', 'billing_details.vat_number'], + format: 'csv', + where: { + title: { equals: 'Snake Case Round Trip' }, + }, + }, + }) + + await payload.jobs.run() + + const exportedDoc = await payload.findByID({ + collection: 'exports', + id: exportDoc.id, + }) + + const csvPath = path.join(dirname, './uploads', exportedDoc.filename as string) + const csv = fs.readFileSync(csvPath, 'utf-8') + + expect(csv).toContain('vat_number') + expect(csv).toContain('billing_details_vat_number') + + const importDoc = await importCSV(csv.replace(/IT12345678901/, 'IT55555555555')) + + expect(importDoc.status).toBe('completed') + expect(importDoc.summary?.issues).toBe(0) + + const updated = await payload.findByID({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + id: post.id, + }) + + expect(updated.vat_number).toBe('IT55555555555') + expect(updated.billing_details?.vat_number).toBe('IT99999999999') + }) + }) }) diff --git a/test/plugin-import-export/payload-types.ts b/test/plugin-import-export/payload-types.ts index fd249a78cd8..d69be4d353f 100644 --- a/test/plugin-import-export/payload-types.ts +++ b/test/plugin-import-export/payload-types.ts @@ -63,15 +63,15 @@ export type SupportedTimezones = | 'UTC'; /** * This interface was referenced by `Config`'s JSON-Schema - * via the `definition` "LexicalNodes_0DD453D3". + * via the `definition` "LexicalNodes_BDA6866C". */ -export type LexicalNodes_0DD453D3 = +export type LexicalNodes_BDA6866C = | SerializedTextNode | SerializedTabNode | SerializedLineBreakNode - | SerializedParagraphNode + | SerializedParagraphNode | SerializedBlockNode - | SerializedHeadingNode + | SerializedHeadingNode | SerializedUploadNode<'media', LexicalUploadFields_1AB4670B> | SerializedUploadNode<'exports'> | SerializedUploadNode<'posts-export'> @@ -88,11 +88,11 @@ export type LexicalNodes_0DD453D3 = | SerializedUploadNode<'posts-with-hooks-import'> | SerializedUploadNode<'posts-with-field-hooks-import'> | SerializedUploadNode<'posts-with-column-map-import'> - | SerializedQuoteNode - | SerializedListNode - | SerializedListItemNode - | SerializedAutoLinkNode - | SerializedLinkNode + | SerializedQuoteNode + | SerializedListNode + | SerializedListItemNode + | SerializedAutoLinkNode + | SerializedLinkNode | SerializedRelationshipNode< | 'users' | 'pages' @@ -105,6 +105,7 @@ export type LexicalNodes_0DD453D3 = | 'posts-with-hooks' | 'posts-with-field-hooks' | 'posts-with-column-map' + | 'posts-with-snake-case-fields' | 'custom-id-pages' | 'payload-kv' | 'payload-jobs' @@ -130,6 +131,7 @@ export interface Config { 'posts-with-hooks': PostsWithHook; 'posts-with-field-hooks': PostsWithFieldHook; 'posts-with-column-map': PostsWithColumnMap; + 'posts-with-snake-case-fields': PostsWithSnakeCaseField; media: Media; 'custom-id-pages': CustomIdPage; exports: Export; @@ -166,6 +168,7 @@ export interface Config { 'posts-with-hooks': PostsWithHooksSelect | PostsWithHooksSelect; 'posts-with-field-hooks': PostsWithFieldHooksSelect | PostsWithFieldHooksSelect; 'posts-with-column-map': PostsWithColumnMapSelect | PostsWithColumnMapSelect; + 'posts-with-snake-case-fields': PostsWithSnakeCaseFieldsSelect | PostsWithSnakeCaseFieldsSelect; media: MediaSelect | MediaSelect; 'custom-id-pages': CustomIdPagesSelect | CustomIdPagesSelect; exports: ExportsSelect | ExportsSelect; @@ -198,11 +201,17 @@ export interface Config { | null | ('en' | 'es' | 'de' | 'he') | ('en' | 'es' | 'de' | 'he')[]; - globals: {}; - globalsSelect: {}; + globals: { + 'payload-jobs-stats': PayloadJobsStat; + }; + globalsSelect: { + 'payload-jobs-stats': PayloadJobsStatsSelect | PayloadJobsStatsSelect; + }; locale: 'en' | 'es' | 'de' | 'he'; widgets: { collections: CollectionsWidget; + 'collection-query': CollectionQueryWidget; + activity: ActivityWidget; }; user: User; jobs: { @@ -314,7 +323,7 @@ export interface Page { | number | boolean | null; - richTextField?: LexicalRichText | null; + richTextField?: LexicalRichText | null; relationship?: (string | null) | User; excerpt?: string | null; /** @@ -382,7 +391,7 @@ export interface Hero { * via the `definition` "Content". */ export interface Content { - richText?: LexicalRichText | null; + richText?: LexicalRichText | null; id?: string | null; blockName?: string | null; blockType: 'content'; @@ -395,7 +404,7 @@ export interface FaqSection { faqs?: | { question?: string | null; - answer?: LexicalRichText | null; + answer?: LexicalRichText | null; id?: string | null; }[] | null; @@ -410,7 +419,7 @@ export interface FaqSection { export interface Post { id: string; title: string; - content?: LexicalRichText | null; + content?: LexicalRichText | null; updatedAt: string; createdAt: string; _status?: ('draft' | 'published') | null; @@ -441,7 +450,7 @@ export interface Media { export interface PostsExportsOnly { id: string; title: string; - content?: LexicalRichText | null; + content?: LexicalRichText | null; updatedAt: string; createdAt: string; _status?: ('draft' | 'published') | null; @@ -453,7 +462,7 @@ export interface PostsExportsOnly { export interface PostsImportsOnly { id: string; title: string; - content?: LexicalRichText | null; + content?: LexicalRichText | null; updatedAt: string; createdAt: string; _status?: ('draft' | 'published') | null; @@ -465,7 +474,7 @@ export interface PostsImportsOnly { export interface PostsNoJobsQueue { id: string; title: string; - content?: LexicalRichText | null; + content?: LexicalRichText | null; updatedAt: string; createdAt: string; _status?: ('draft' | 'published') | null; @@ -562,6 +571,27 @@ export interface PostsWithColumnMap { updatedAt: string; createdAt: string; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "posts-with-snake-case-fields". + */ +export interface PostsWithSnakeCaseField { + id: string; + title?: string | null; + vat_number?: string | null; + billing_details?: { + vat_number?: string | null; + }; + line_items?: + | { + item_code?: string | null; + id?: string | null; + }[] + | null; + localized_note?: string | null; + updatedAt: string; + createdAt: string; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "custom-id-pages". @@ -1179,6 +1209,15 @@ export interface PayloadJob { | number | boolean | null; + meta?: + | { + [k: string]: unknown; + } + | unknown[] + | string + | number + | boolean + | null; completedAt?: string | null; totalTried?: number | null; /** @@ -1240,7 +1279,8 @@ export interface PayloadJob { taskSlug?: ('inline' | 'createCollectionExport' | 'createCollectionImport') | null; queue?: string | null; waitUntil?: string | null; - processing?: boolean | null; + processingUntil?: string | null; + processingToken?: string | null; updatedAt: string; createdAt: string; } @@ -1295,6 +1335,10 @@ export interface PayloadLockedDocument { relationTo: 'posts-with-column-map'; value: string | PostsWithColumnMap; } | null) + | ({ + relationTo: 'posts-with-snake-case-fields'; + value: string | PostsWithSnakeCaseField; + } | null) | ({ relationTo: 'media'; value: string | Media; @@ -1604,6 +1648,28 @@ export interface PostsWithColumnMapSelect { updatedAt?: T; createdAt?: T; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "posts-with-snake-case-fields_select". + */ +export interface PostsWithSnakeCaseFieldsSelect { + title?: T; + vat_number?: T; + billing_details?: + | T + | { + vat_number?: T; + }; + line_items?: + | T + | { + item_code?: T; + id?: T; + }; + localized_note?: T; + updatedAt?: T; + createdAt?: T; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "media_select". @@ -2089,6 +2155,7 @@ export interface PayloadKvSelect { export interface PayloadJobsSelect { input?: T; taskStatus?: T; + meta?: T; completedAt?: T; totalTried?: T; hasError?: T; @@ -2109,7 +2176,8 @@ export interface PayloadJobsSelect { taskSlug?: T; queue?: T; waitUntil?: T; - processing?: T; + processingUntil?: T; + processingToken?: T; updatedAt?: T; createdAt?: T; } @@ -2145,6 +2213,34 @@ export interface PayloadMigrationsSelect { updatedAt?: T; createdAt?: T; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "payload-jobs-stats". + */ +export interface PayloadJobsStat { + id: string; + stats?: + | { + [k: string]: unknown; + } + | unknown[] + | string + | number + | boolean + | null; + updatedAt?: string | null; + createdAt?: string | null; +} +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "payload-jobs-stats_select". + */ +export interface PayloadJobsStatsSelect { + stats?: T; + updatedAt?: T; + createdAt?: T; + globalType?: T; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "collections_widget". @@ -2155,6 +2251,102 @@ export interface CollectionsWidget { }; width: 'full'; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "collection-query_widget". + */ +export interface CollectionQueryWidget { + data?: { + title?: string | null; + relatedCollection: + | 'users' + | 'pages' + | 'posts' + | 'posts-exports-only' + | 'posts-imports-only' + | 'posts-no-jobs-queue' + | 'posts-with-limits' + | 'posts-with-s3' + | 'posts-with-hooks' + | 'posts-with-field-hooks' + | 'posts-with-column-map' + | 'posts-with-snake-case-fields' + | 'media' + | 'custom-id-pages' + | 'exports' + | 'posts-export' + | 'posts-no-jobs-queue-export' + | 'posts-with-s3-export' + | 'posts-with-limits-export' + | 'posts-with-hooks-export' + | 'posts-with-field-hooks-export' + | 'posts-with-column-map-export' + | 'imports' + | 'posts-import' + | 'posts-with-s3-import' + | 'posts-with-limits-import' + | 'posts-with-hooks-import' + | 'posts-with-field-hooks-import' + | 'posts-with-column-map-import' + | 'payload-jobs'; + where?: + | { + [k: string]: unknown; + } + | unknown[] + | string + | number + | boolean + | null; + sortField?: string | null; + sortDirection?: ('asc' | 'desc') | null; + limit?: number | null; + }; + width: 'x-small' | 'small' | 'medium' | 'large' | 'x-large' | 'full'; +} +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "activity_widget". + */ +export interface ActivityWidget { + data?: { + excludedCollections?: + | ( + | 'users' + | 'pages' + | 'posts' + | 'posts-exports-only' + | 'posts-imports-only' + | 'posts-no-jobs-queue' + | 'posts-with-limits' + | 'posts-with-s3' + | 'posts-with-hooks' + | 'posts-with-field-hooks' + | 'posts-with-column-map' + | 'posts-with-snake-case-fields' + | 'media' + | 'custom-id-pages' + | 'exports' + | 'posts-export' + | 'posts-no-jobs-queue-export' + | 'posts-with-s3-export' + | 'posts-with-limits-export' + | 'posts-with-hooks-export' + | 'posts-with-field-hooks-export' + | 'posts-with-column-map-export' + | 'imports' + | 'posts-import' + | 'posts-with-s3-import' + | 'posts-with-limits-import' + | 'posts-with-hooks-import' + | 'posts-with-field-hooks-import' + | 'posts-with-column-map-import' + | 'payload-jobs' + )[] + | null; + }; + width: 'x-small' | 'small' | 'medium' | 'large' | 'x-large' | 'full'; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "TaskCreateCollectionExport". @@ -2176,6 +2368,7 @@ export interface TaskCreateCollectionExport { | 'posts-with-hooks' | 'posts-with-field-hooks' | 'posts-with-column-map' + | 'posts-with-snake-case-fields' | 'media' | 'custom-id-pages' | 'exports' diff --git a/test/plugin-import-export/shared.ts b/test/plugin-import-export/shared.ts index 149e8f2d777..a7b22c13816 100644 --- a/test/plugin-import-export/shared.ts +++ b/test/plugin-import-export/shared.ts @@ -29,3 +29,5 @@ export const postsWithHooksImportSlug = 'posts-with-hooks-import' export const postsWithFieldHooksSlug = 'posts-with-field-hooks' export const postsWithColumnMapSlug = 'posts-with-column-map' + +export const postsWithSnakeCaseFieldsSlug = 'posts-with-snake-case-fields'