diff --git a/packages/plugin-import-export/src/utilities/flatKeyToPathSegments.spec.ts b/packages/plugin-import-export/src/utilities/flatKeyToPathSegments.spec.ts new file mode 100644 index 00000000000..a53c2a0553d --- /dev/null +++ b/packages/plugin-import-export/src/utilities/flatKeyToPathSegments.spec.ts @@ -0,0 +1,400 @@ +import type { FlattenedBlock, FlattenedField } from 'payload' + +import { flatKeyToPathSegments } from './flatKeyToPathSegments.js' + +import { describe, it, expect } from 'vitest' + +describe('flatKeyToPathSegments', () => { + describe('flat field names containing underscores', () => { + const fields: FlattenedField[] = [ + { name: 'vat_number', type: 'text' } as FlattenedField, + { name: '_start_with_underscore', type: 'text' } as FlattenedField, + { name: 'with_numbers_1', type: 'text' } as FlattenedField, + ] + + it('should keep a snake_case name as a single segment', () => { + expect(flatKeyToPathSegments({ fields, flatKey: 'vat_number' })).toEqual(['vat_number']) + }) + + it('should keep a leading underscore', () => { + expect(flatKeyToPathSegments({ fields, flatKey: '_start_with_underscore' })).toEqual([ + '_start_with_underscore', + ]) + }) + + it('should not treat a trailing digit in a name as an array index', () => { + expect(flatKeyToPathSegments({ fields, flatKey: 'with_numbers_1' })).toEqual([ + 'with_numbers_1', + ]) + }) + }) + + describe('groups', () => { + it('should resolve a snake_case field nested in a group', () => { + const fields: FlattenedField[] = [ + { + name: 'group', + type: 'group', + flattenedFields: [{ name: 'vat_number', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'group_vat_number' })).toEqual([ + 'group', + 'vat_number', + ]) + }) + + it('should resolve a group whose own name contains underscores', () => { + const fields: FlattenedField[] = [ + { + name: 'billing_details', + type: 'group', + flattenedFields: [{ name: 'vat_number', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'billing_details_vat_number' })).toEqual([ + 'billing_details', + 'vat_number', + ]) + }) + + it('should prefer a flat field over descending into a group with a colliding name', () => { + const fields: FlattenedField[] = [ + { name: 'vat_number', type: 'text' } as FlattenedField, + { + name: 'vat', + type: 'group', + flattenedFields: [{ name: 'number', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'vat_number' })).toEqual(['vat_number']) + }) + + it('should backtrack to a group when the longer flat name cannot resolve the whole key', () => { + const fields: FlattenedField[] = [ + { name: 'vat_number', type: 'text' } as FlattenedField, + { + name: 'vat', + type: 'group', + flattenedFields: [{ name: 'number_suffix', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'vat_number_suffix' })).toEqual([ + 'vat', + 'number_suffix', + ]) + }) + + it('should resolve nested groups that both contain underscores', () => { + const fields: FlattenedField[] = [ + { + name: 'outer_group', + type: 'group', + flattenedFields: [ + { + name: 'inner_group', + type: 'group', + flattenedFields: [{ name: 'my_field', type: 'text' }], + }, + ], + } as unknown as FlattenedField, + ] + + expect( + flatKeyToPathSegments({ fields, flatKey: 'outer_group_inner_group_my_field' }), + ).toEqual(['outer_group', 'inner_group', 'my_field']) + }) + + it('should resolve a named tab like a group', () => { + const fields: FlattenedField[] = [ + { + name: 'meta_tab', + type: 'tab', + flattenedFields: [{ name: 'vat_number', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'meta_tab_vat_number' })).toEqual([ + 'meta_tab', + 'vat_number', + ]) + }) + + it('should not resolve a group that carries no flattenedFields', () => { + const fields: FlattenedField[] = [{ name: 'group', type: 'group' } as FlattenedField] + + expect(flatKeyToPathSegments({ fields, flatKey: 'group_field1' })).toBeUndefined() + }) + }) + + describe('localized fields', () => { + const localeCodes = ['en', 'en_US', 'es'] + + it('should emit the locale as its own segment', () => { + const fields: FlattenedField[] = [ + { name: 'vat_number', type: 'text', localized: true } as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'vat_number_en', localeCodes })).toEqual([ + 'vat_number', + 'en', + ]) + }) + + it('should prefer a longer locale code over a shorter one', () => { + const fields: FlattenedField[] = [ + { name: 'title', type: 'text', localized: true } as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'title_en_US', localeCodes })).toEqual([ + 'title', + 'en_US', + ]) + }) + + it('should recognize a locale by shape when no locale codes are configured', () => { + const fields: FlattenedField[] = [ + { name: 'vat_number', type: 'text', localized: true } as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'vat_number_es' })).toEqual([ + 'vat_number', + 'es', + ]) + }) + + it('should resolve a field inside a localized group', () => { + const fields: FlattenedField[] = [ + { + name: 'group', + type: 'group', + localized: true, + flattenedFields: [{ name: 'vat_number', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect( + flatKeyToPathSegments({ fields, flatKey: 'group_en_vat_number', localeCodes }), + ).toEqual(['group', 'en', 'vat_number']) + }) + + it('should backtrack when a locale-shaped segment is actually part of a field name', () => { + const fields: FlattenedField[] = [ + { + name: 'group', + type: 'group', + localized: true, + flattenedFields: [{ name: 'es_label', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'group_es_label' })).toEqual([ + 'group', + 'es_label', + ]) + }) + + it('should not consume a locale suffix on a field that is not localized', () => { + const fields: FlattenedField[] = [{ name: 'title', type: 'text' } as FlattenedField] + + expect(flatKeyToPathSegments({ fields, flatKey: 'title_en', localeCodes })).toBeUndefined() + }) + }) + + describe('arrays and hasMany fields', () => { + it('should resolve a snake_case field inside an array row', () => { + const fields: FlattenedField[] = [ + { + name: 'line_items', + type: 'array', + flattenedFields: [{ name: 'vat_number', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'line_items_0_vat_number' })).toEqual([ + 'line_items', + '0', + 'vat_number', + ]) + }) + + it('should resolve an indexed hasMany scalar column', () => { + const fields: FlattenedField[] = [ + { name: 'my_tags', type: 'text', hasMany: true } as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'my_tags_0' })).toEqual(['my_tags', '0']) + }) + + it('should not treat an index suffix as valid on a field that is not hasMany', () => { + const fields: FlattenedField[] = [{ name: 'my_tags', type: 'text' } as FlattenedField] + + expect(flatKeyToPathSegments({ fields, flatKey: 'my_tags_0' })).toBeUndefined() + }) + }) + + describe('blocks', () => { + const fields: FlattenedField[] = [ + { + name: 'page_content', + type: 'blocks', + blocks: [ + { + slug: 'hero_section', + flattenedFields: [{ name: 'call_to_action', type: 'text' }], + }, + ], + } as unknown as FlattenedField, + ] + + it('should resolve an underscored block slug and field name', () => { + expect( + flatKeyToPathSegments({ fields, flatKey: 'page_content_0_hero_section_call_to_action' }), + ).toEqual(['page_content', '0', 'hero_section', 'call_to_action']) + }) + + it('should resolve the blockType column', () => { + expect( + flatKeyToPathSegments({ fields, flatKey: 'page_content_0_hero_section_blockType' }), + ).toEqual(['page_content', '0', 'hero_section', 'blockType']) + }) + + it('should resolve a block referenced by slug', () => { + const referencingFields: FlattenedField[] = [ + { + name: 'page_content', + type: 'blocks', + blocks: ['hero_section'], + } as unknown as FlattenedField, + ] + + const blocksBySlug: Record = { + hero_section: { + slug: 'hero_section', + fields: [], + flattenedFields: [{ name: 'call_to_action', type: 'text' }], + } as unknown as FlattenedBlock, + } + + expect( + flatKeyToPathSegments({ + blocksBySlug, + fields: referencingFields, + flatKey: 'page_content_0_hero_section_call_to_action', + }), + ).toEqual(['page_content', '0', 'hero_section', 'call_to_action']) + }) + + it('should not resolve a block reference that cannot be looked up', () => { + const referencingFields: FlattenedField[] = [ + { + name: 'page_content', + type: 'blocks', + blocks: ['hero_section'], + } as unknown as FlattenedField, + ] + + expect( + flatKeyToPathSegments({ + fields: referencingFields, + flatKey: 'page_content_0_hero_section_call_to_action', + }), + ).toBeUndefined() + }) + }) + + describe('relationships', () => { + it('should resolve the id and relationTo columns of a polymorphic relationship', () => { + const fields: FlattenedField[] = [ + { + name: 'related_doc', + type: 'relationship', + relationTo: ['posts', 'pages'], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'related_doc_id' })).toEqual([ + 'related_doc', + 'id', + ]) + expect(flatKeyToPathSegments({ fields, flatKey: 'related_doc_relationTo' })).toEqual([ + 'related_doc', + 'relationTo', + ]) + }) + + it('should resolve indexed columns of a polymorphic hasMany relationship', () => { + const fields: FlattenedField[] = [ + { + name: 'related_docs', + type: 'relationship', + hasMany: true, + relationTo: ['posts', 'pages'], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'related_docs_0_relationTo' })).toEqual([ + 'related_docs', + '0', + 'relationTo', + ]) + }) + + it('should not resolve populated relationship data', () => { + const fields: FlattenedField[] = [ + { name: 'author', type: 'relationship', relationTo: 'users' } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields, flatKey: 'author_email_address' })).toBeUndefined() + }) + }) + + describe('unresolvable keys', () => { + const fields: FlattenedField[] = [{ name: 'title', type: 'text' } as FlattenedField] + + it('should not resolve a column that is absent from the schema', () => { + expect(flatKeyToPathSegments({ fields, flatKey: 'my_extra_column' })).toBeUndefined() + }) + + it('should not resolve a partial field path', () => { + const groupFields: FlattenedField[] = [ + { + name: 'group', + type: 'group', + flattenedFields: [{ name: 'nested', type: 'text' }], + } as unknown as FlattenedField, + ] + + expect(flatKeyToPathSegments({ fields: groupFields, flatKey: 'group' })).toBeUndefined() + }) + }) + + it('should match a naive underscore split when no field name contains an underscore', () => { + const fields: FlattenedField[] = [ + { name: 'title', type: 'text', localized: true } as FlattenedField, + { + name: 'items', + type: 'array', + flattenedFields: [{ name: 'name', type: 'text' }], + } as unknown as FlattenedField, + { + name: 'group', + type: 'group', + flattenedFields: [{ name: 'nested', type: 'text' }], + } as unknown as FlattenedField, + { name: 'poly', type: 'relationship', relationTo: ['posts'] } as unknown as FlattenedField, + ] + + const keys = ['title_en', 'items_0_name', 'group_nested', 'poly_id', 'poly_relationTo'] + + for (const flatKey of keys) { + expect(flatKeyToPathSegments({ fields, flatKey, localeCodes: ['en'] })).toEqual( + flatKey.split('_'), + ) + } + }) +}) diff --git a/packages/plugin-import-export/src/utilities/flatKeyToPathSegments.ts b/packages/plugin-import-export/src/utilities/flatKeyToPathSegments.ts new file mode 100644 index 00000000000..3fdd967d64e --- /dev/null +++ b/packages/plugin-import-export/src/utilities/flatKeyToPathSegments.ts @@ -0,0 +1,306 @@ +import type { FlattenedBlock, FlattenedField } from 'payload' + +import { getBlockFlattenedFields, getNestedFlattenedFields } from './flattenedFields.js' + +type Args = { + /** + * Resolved block definitions, used to look up blocks that a `blocks` field + * references by slug rather than defines inline. Pass `req.payload.blocks`. + */ + blocksBySlug?: Record + fields: FlattenedField[] + flatKey: string + /** + * Configured locale codes. When omitted, a locale segment is recognized by + * shape (`en`, `en_US`) instead — the same heuristic `unflattenPostProcess` uses. + */ + localeCodes?: string[] +} + +type ResolveContext = Pick + +/** `null` rest means the prefix consumed the whole key. */ +type Taken = { + rest: null | string +} + +const indexSegment = /^\d+$/ + +const localeSegment = /^[a-z]{2}(?:_[A-Z]{2})?$/ + +const segmentsCache = new WeakMap>() + +/** + * Resolves a flattened CSV column name into the path segments that address it + * within a document, using the field schema rather than assuming every + * underscore is a nesting separator. `vat_number` resolves to `['vat_number']`, + * while `group_vat_number` resolves to `['group', 'vat_number']`. + * + * Array indices, block slugs and locale codes are emitted as their own segments, + * so a key the naive `split('_')` already handled correctly resolves to the + * identical segments. + * + * Returns `undefined` when the schema cannot account for the key — extra CSV + * columns, or the populated relationship data an export at depth > 0 produces. + * Callers are expected to fall back to `flatKey.split('_')` in that case. + * + * Field names are matched longest-first with backtracking, so a flat field named + * `vat_number` wins over a `vat` group containing `number`. Such a schema is + * ambiguous on export too — both produce a single `vat_number` column — so the + * collision cannot be resolved from the CSV alone. + */ +export const flatKeyToPathSegments = ({ + blocksBySlug, + fields, + flatKey, + localeCodes, +}: Args): string[] | undefined => { + let cached = segmentsCache.get(fields) + + if (!cached) { + cached = new Map() + segmentsCache.set(fields, cached) + } + + // Resolution depends on the locales as well as the schema, and a column set is + // re-resolved for every imported row — worth memoizing across rows. + const cacheKey = `${localeCodes?.join(',') ?? ''}:${flatKey}` + const memoized = cached.get(cacheKey) + + if (memoized !== undefined) { + return memoized ?? undefined + } + + const resolved = resolveInFields(fields, flatKey, { blocksBySlug, localeCodes }) + cached.set(cacheKey, resolved ?? null) + + return resolved +} + +const resolveInFields = ( + fields: FlattenedField[], + key: string, + ctx: ResolveContext, +): string[] | undefined => { + const candidates = fields + .filter((field): field is { name: string } & FlattenedField => + Boolean('name' in field && field.name), + ) + .sort((a, b) => b.name.length - a.name.length) + + for (const field of candidates) { + const taken = takePrefix(key, field.name) + if (!taken) { + continue + } + + const suffix = resolveField({ ctx, field, tail: taken.rest }) + if (suffix) { + return [field.name, ...suffix] + } + } + + return undefined +} + +/** + * Consumes an optional locale segment before the field's own suffix handling, so + * both `title_en` and localized-group keys like `group_en_vat_number` resolve. + * Falls back to treating the tail as part of the field when the locale reading + * doesn't resolve. + */ +const resolveField = ({ + ctx, + field, + tail, +}: { + ctx: ResolveContext + field: FlattenedField + tail: null | string +}): string[] | undefined => { + if (field.localized && tail !== null) { + const taken = takeLocale(tail, ctx.localeCodes) + + if (taken) { + const suffix = resolveFieldBody({ ctx, field, tail: taken.rest }) + if (suffix) { + return [taken.locale, ...suffix] + } + } + } + + return resolveFieldBody({ ctx, field, tail }) +} + +const resolveFieldBody = ({ + ctx, + field, + tail, +}: { + ctx: ResolveContext + field: FlattenedField + tail: null | string +}): string[] | undefined => { + switch (field.type) { + case 'array': { + if (tail === null) { + return undefined + } + + const [index, rest] = splitFirstSegment(tail) + if (!indexSegment.test(index)) { + return undefined + } + if (rest === null) { + return [index] + } + + const suffix = resolveInFields(getNestedFlattenedFields(field) ?? [], rest, ctx) + + return suffix && [index, ...suffix] + } + + case 'blocks': { + if (tail === null) { + return undefined + } + + const [index, rest] = splitFirstSegment(tail) + if (!indexSegment.test(index) || rest === null) { + return undefined + } + + return resolveInBlocks({ ctx, field, index, key: rest }) + } + + case 'group': + case 'tab': { + if (tail === null) { + return undefined + } + + return resolveInFields(getNestedFlattenedFields(field) ?? [], tail, ctx) + } + + case 'relationship': + case 'upload': { + if (tail === null) { + return [] + } + + const [index, rest] = splitFirstSegment(tail) + + // hasMany: `rel_0`, or `rel_0_id` / `rel_0_relationTo` when polymorphic + if (indexSegment.test(index)) { + if (rest === null) { + return [index] + } + + return isPolymorphicSuffix(rest) ? [index, rest] : undefined + } + + // hasOne polymorphic: `rel_id` / `rel_relationTo` + return isPolymorphicSuffix(tail) ? [tail] : undefined + } + + default: { + if (tail === null) { + return [] + } + + // hasMany scalars are exported one column per index, e.g. `tags_0` + const isHasMany = 'hasMany' in field && field.hasMany === true + + return isHasMany && indexSegment.test(tail) ? [tail] : undefined + } + } +} + +const resolveInBlocks = ({ + ctx, + field, + index, + key, +}: { + ctx: ResolveContext + field: FlattenedField + index: string + key: string +}): string[] | undefined => { + const blocks = 'blocks' in field ? (field.blocks ?? []) : [] + + const definitions = blocks + .map((block) => (typeof block === 'string' ? ctx.blocksBySlug?.[block] : block)) + .filter((block): block is FlattenedBlock => Boolean(block)) + .sort((a, b) => b.slug.length - a.slug.length) + + for (const block of definitions) { + const taken = takePrefix(key, block.slug) + if (!taken || taken.rest === null) { + continue + } + + if (taken.rest === 'blockType') { + return [index, block.slug, 'blockType'] + } + + const suffix = resolveInFields(getBlockFlattenedFields(block), taken.rest, ctx) + if (suffix) { + return [index, block.slug, ...suffix] + } + } + + return undefined +} + +const takePrefix = (key: string, prefix: string): Taken | undefined => { + if (key === prefix) { + return { rest: null } + } + + if (key.startsWith(`${prefix}_`)) { + return { rest: key.slice(prefix.length + 1) } + } + + return undefined +} + +const takeLocale = ( + key: string, + localeCodes: string[] | undefined, +): ({ locale: string } & Taken) | undefined => { + for (const candidate of localeCandidates(key, localeCodes)) { + const taken = takePrefix(key, candidate) + if (taken) { + return { locale: candidate, rest: taken.rest } + } + } + + return undefined +} + +/** + * Locale codes to try against the head of `key`, longest first so `en_US` wins + * over a co-existing `en`. Without configured codes, the first one or two + * segments are offered when they look like a locale. + */ +const localeCandidates = (key: string, localeCodes: string[] | undefined): string[] => { + if (localeCodes && localeCodes.length > 0) { + return [...localeCodes].sort((a, b) => b.length - a.length) + } + + const [first, second] = key.split('_') + + return [second === undefined ? undefined : `${first}_${second}`, first].filter( + (candidate): candidate is string => candidate !== undefined && localeSegment.test(candidate), + ) +} + +const splitFirstSegment = (key: string): [string, null | string] => { + const boundary = key.indexOf('_') + + return boundary === -1 ? [key, null] : [key.slice(0, boundary), key.slice(boundary + 1)] +} + +const isPolymorphicSuffix = (segment: string): boolean => + segment === 'id' || segment === 'relationTo' diff --git a/packages/plugin-import-export/src/utilities/unflattenObject.spec.ts b/packages/plugin-import-export/src/utilities/unflattenObject.spec.ts index 681f1f35359..70d39b30b8b 100644 --- a/packages/plugin-import-export/src/utilities/unflattenObject.spec.ts +++ b/packages/plugin-import-export/src/utilities/unflattenObject.spec.ts @@ -367,6 +367,165 @@ describe('unflattenObject', () => { }) }) + describe('field names containing underscores', () => { + const fields: FlattenedField[] = [ + { + name: 'simple', + type: 'text', + } as FlattenedField, + { + name: 'camelCase', + type: 'text', + } as FlattenedField, + { + name: 'PascalCase', + type: 'text', + } as FlattenedField, + { + name: 'with_underscores', + type: 'text', + } as FlattenedField, + { + name: '_start_with_underscore', + type: 'text', + } as FlattenedField, + { + name: 'with_numbers_1', + type: 'text', + } as FlattenedField, + { + name: 'localized_with_underscores_with_numbers_1', + type: 'text', + localized: true, + } as FlattenedField, + ] + + it('should keep a snake_case field flat instead of nesting it', () => { + const data = { + vat_number: 'IT12345678901', + } + + const result = unflattenObject({ + data, + fields: [{ name: 'vat_number', type: 'text' } as FlattenedField], + req: mockReq, + }) + + expect(result).toEqual({ + vat_number: 'IT12345678901', + }) + }) + + it('should handle all permitted field name shapes at the top level', () => { + const data = { + simple: 'simple', + camelCase: 'camelCase', + PascalCase: 'PascalCase', + with_underscores: 'with_underscores', + _start_with_underscore: '_start_with_underscore', + with_numbers_1: 'with_numbers_1', + localized_with_underscores_with_numbers_1_en: + 'localized_with_underscores_with_numbers_1_en', + localized_with_underscores_with_numbers_1_es: + 'localized_with_underscores_with_numbers_1_es', + } + + const result = unflattenObject({ data, fields, req: mockReq }) + + expect(result).toEqual({ + simple: 'simple', + camelCase: 'camelCase', + PascalCase: 'PascalCase', + with_underscores: 'with_underscores', + _start_with_underscore: '_start_with_underscore', + with_numbers_1: 'with_numbers_1', + localized_with_underscores_with_numbers_1: { + en: 'localized_with_underscores_with_numbers_1_en', + es: 'localized_with_underscores_with_numbers_1_es', + }, + }) + }) + + it('should handle all permitted field name shapes nested in a group', () => { + const groupFields: FlattenedField[] = [ + { + name: 'group', + type: 'group', + flattenedFields: fields, + } as unknown as FlattenedField, + ] + + const data = { + group_simple: 'simple', + group_camelCase: 'camelCase', + group_PascalCase: 'PascalCase', + group_with_underscores: 'with_underscores', + group__start_with_underscore: '_start_with_underscore', + group_with_numbers_1: 'with_numbers_1', + group_localized_with_underscores_with_numbers_1_en: + 'localized_with_underscores_with_numbers_1_en', + group_localized_with_underscores_with_numbers_1_es: + 'localized_with_underscores_with_numbers_1_es', + } + + const result = unflattenObject({ data, fields: groupFields, req: mockReq }) + + expect(result).toEqual({ + group: { + simple: 'simple', + camelCase: 'camelCase', + PascalCase: 'PascalCase', + with_underscores: 'with_underscores', + _start_with_underscore: '_start_with_underscore', + with_numbers_1: 'with_numbers_1', + localized_with_underscores_with_numbers_1: { + en: 'localized_with_underscores_with_numbers_1_en', + es: 'localized_with_underscores_with_numbers_1_es', + }, + }, + }) + }) + + it('should handle a snake_case field inside an array row', () => { + const arrayFields: FlattenedField[] = [ + { + name: 'items', + type: 'array', + flattenedFields: [{ name: 'vat_number', type: 'text' }], + } as unknown as FlattenedField, + ] + + const data = { + items_0_vat_number: 'IT12345678901', + } + + const result = unflattenObject({ data, fields: arrayFields, req: mockReq }) + + expect(result).toEqual({ + items: [{ vat_number: 'IT12345678901' }], + }) + }) + + it('should not treat a text field named with an _id suffix as a relationship', () => { + const idSuffixFields: FlattenedField[] = [ + { + name: 'external_id', + type: 'text', + } as FlattenedField, + ] + + const data = { + external_id: 'abc-123', + } + + const result = unflattenObject({ data, fields: idSuffixFields, req: mockReq }) + + expect(result).toEqual({ + external_id: 'abc-123', + }) + }) + }) + describe('polymorphic relationships', () => { const fields: FlattenedField[] = [ { diff --git a/packages/plugin-import-export/src/utilities/unflattenObject.ts b/packages/plugin-import-export/src/utilities/unflattenObject.ts index a079c51b18d..a127e8c3346 100644 --- a/packages/plugin-import-export/src/utilities/unflattenObject.ts +++ b/packages/plugin-import-export/src/utilities/unflattenObject.ts @@ -2,6 +2,7 @@ import type { FlattenedField, PayloadRequest } from 'payload' import type { ImportFieldHookEntry } from '../types.js' +import { flatKeyToPathSegments } from './flatKeyToPathSegments.js' import { getBlockFlattenedFields, getNestedFlattenedFields } from './flattenedFields.js' import { postProcessDocument } from './unflattenPostProcess.js' @@ -79,6 +80,14 @@ export const unflattenObject = ({ const arrayLikeNames = new Set() collectArrayLikeNames(fields, arrayLikeNames) + const localization = req?.payload?.config?.localization + + const segmentArgs = { + blocksBySlug: req?.payload?.blocks, + fields, + localeCodes: localization ? localization.localeCodes : undefined, + } + // Sort keys to ensure array indices are processed in order const sortedKeys = Object.keys(data).sort((a, b) => { // Extract array indices from flattened keys (e.g., "field_0_subfield" -> "0") @@ -184,7 +193,10 @@ export const unflattenObject = ({ } // Example: "blocks_0_content_text" -> ["blocks", "0", "content", "text"] - const pathSegments = flatKey.split('_') + // Resolved against the field schema so names that contain underscores stay + // intact ("vat_number"), falling back to a naive split for keys the schema + // cannot account for — extra CSV columns, or populated relationship data. + const pathSegments = flatKeyToPathSegments({ ...segmentArgs, flatKey }) ?? flatKey.split('_') let currentObject: Record = result for (let i = 0; i < pathSegments.length; i++) { diff --git a/test/plugin-import-export/collections/PostsWithSnakeCaseFields.ts b/test/plugin-import-export/collections/PostsWithSnakeCaseFields.ts new file mode 100644 index 00000000000..8226a0ee292 --- /dev/null +++ b/test/plugin-import-export/collections/PostsWithSnakeCaseFields.ts @@ -0,0 +1,49 @@ +import type { CollectionConfig } from 'payload' + +import { postsWithSnakeCaseFieldsSlug } from '../shared.js' + +export const PostsWithSnakeCaseFields: CollectionConfig = { + slug: postsWithSnakeCaseFieldsSlug, + labels: { + singular: 'Posts With Snake Case Fields', + plural: 'Posts With Snake Case Fields', + }, + admin: { + useAsTitle: 'title', + }, + fields: [ + { + name: 'title', + type: 'text', + }, + { + name: 'vat_number', + type: 'text', + }, + { + name: 'billing_details', + type: 'group', + fields: [ + { + name: 'vat_number', + type: 'text', + }, + ], + }, + { + name: 'line_items', + type: 'array', + fields: [ + { + name: 'item_code', + type: 'text', + }, + ], + }, + { + name: 'localized_note', + type: 'text', + localized: true, + }, + ], +} diff --git a/test/plugin-import-export/config.ts b/test/plugin-import-export/config.ts index 86dc1c4080e..381328d3e06 100644 --- a/test/plugin-import-export/config.ts +++ b/test/plugin-import-export/config.ts @@ -26,6 +26,7 @@ import { PostsWithFieldHooks } from './collections/PostsWithFieldHooks.js' import { PostsWithHooks } from './collections/PostsWithHooks.js' import { PostsWithLimits } from './collections/PostsWithLimits.js' import { PostsWithS3 } from './collections/PostsWithS3.js' +import { PostsWithSnakeCaseFields } from './collections/PostsWithSnakeCaseFields.js' import { Users } from './collections/Users.js' import { exportAfterHook, @@ -71,6 +72,7 @@ export default buildConfigWithDefaults({ PostsWithHooks, PostsWithFieldHooks, PostsWithColumnMap, + PostsWithSnakeCaseFields, Media, CustomIdPages, ], diff --git a/test/plugin-import-export/int.spec.ts b/test/plugin-import-export/int.spec.ts index 97ab2f1348d..09e121dfa81 100644 --- a/test/plugin-import-export/int.spec.ts +++ b/test/plugin-import-export/int.spec.ts @@ -14,7 +14,7 @@ import { devUser, regularUser } from '../credentials.js' import { clearTestBucket, createTestBucket } from '../storage-s3/test-utils.js' import { readCSV, readJSON } from './helpers.js' import { richTextData } from './seed/richTextData.js' -import { customIdPagesSlug, postsWithS3Slug } from './shared.js' +import { customIdPagesSlug, postsWithS3Slug, postsWithSnakeCaseFieldsSlug } from './shared.js' let payload: Payload let restClient: NextRESTClient @@ -8907,4 +8907,185 @@ describe('@payloadcms/plugin-import-export', () => { posts.docs.forEach((post) => createdPostIDs.push(post.id)) }) }) + + describe('field names containing underscores', () => { + const createdPostIDs: (number | string)[] = [] + + afterEach(async () => { + for (const id of createdPostIDs) { + await payload.delete({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + id, + }) + } + createdPostIDs.length = 0 + }) + + const importCSV = async (csvContent: string) => { + const csvBuffer = Buffer.from(csvContent) + + const importDoc = await payload.create({ + collection: 'imports', + user, + data: { + collectionSlug: postsWithSnakeCaseFieldsSlug, + importMode: 'update', + matchField: 'id', + }, + file: { + data: csvBuffer, + mimetype: 'text/csv', + name: 'snake-case-import-test.csv', + size: csvBuffer.length, + }, + }) + + await payload.jobs.run() + + return payload.findByID({ + collection: 'imports', + id: importDoc.id, + }) + } + + it('should update a snake_case field from CSV', async () => { + const post = await payload.create({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + data: { + title: 'Snake Case Import Test', + vat_number: 'IT00000000000', + }, + }) + + createdPostIDs.push(post.id) + + const importDoc = await importCSV( + `id,title,vat_number\n${post.id},"Snake Case Import Test","IT12345678901"`, + ) + + expect(importDoc.status).toBe('completed') + expect(importDoc.summary?.updated).toBe(1) + expect(importDoc.summary?.issues).toBe(0) + + const updated = await payload.findByID({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + id: post.id, + }) + + expect(updated.vat_number).toBe('IT12345678901') + }) + + it('should update snake_case fields nested in a group and an array from CSV', async () => { + const post = await payload.create({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + data: { + title: 'Snake Case Nested Test', + billing_details: { vat_number: 'IT00000000000' }, + line_items: [{ item_code: 'OLD-1' }], + }, + }) + + createdPostIDs.push(post.id) + + const importDoc = await importCSV( + `id,billing_details_vat_number,line_items_0_item_code\n${post.id},"IT12345678901","NEW-1"`, + ) + + expect(importDoc.status).toBe('completed') + expect(importDoc.summary?.issues).toBe(0) + + const updated = await payload.findByID({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + id: post.id, + }) + + expect(updated.billing_details?.vat_number).toBe('IT12345678901') + expect(updated.line_items?.[0]?.item_code).toBe('NEW-1') + }) + + it('should update a localized snake_case field per locale from CSV', async () => { + const post = await payload.create({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + data: { + title: 'Snake Case Localized Test', + localized_note: 'english original', + }, + }) + + createdPostIDs.push(post.id) + + const importDoc = await importCSV( + `id,localized_note_en,localized_note_es\n${post.id},"english updated","spanish updated"`, + ) + + expect(importDoc.status).toBe('completed') + expect(importDoc.summary?.issues).toBe(0) + + const english = await payload.findByID({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + id: post.id, + locale: 'en', + }) + const spanish = await payload.findByID({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + id: post.id, + locale: 'es', + }) + + expect(english.localized_note).toBe('english updated') + expect(spanish.localized_note).toBe('spanish updated') + }) + + it('should round-trip snake_case fields through export and import', async () => { + const post = await payload.create({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + data: { + title: 'Snake Case Round Trip', + vat_number: 'IT12345678901', + billing_details: { vat_number: 'IT99999999999' }, + }, + }) + + createdPostIDs.push(post.id) + + const exportDoc = await payload.create({ + collection: 'exports', + user, + data: { + collectionSlug: postsWithSnakeCaseFieldsSlug, + fields: ['id', 'vat_number', 'billing_details.vat_number'], + format: 'csv', + where: { + title: { equals: 'Snake Case Round Trip' }, + }, + }, + }) + + await payload.jobs.run() + + const exportedDoc = await payload.findByID({ + collection: 'exports', + id: exportDoc.id, + }) + + const csvPath = path.join(dirname, './uploads', exportedDoc.filename as string) + const csv = fs.readFileSync(csvPath, 'utf-8') + + expect(csv).toContain('vat_number') + expect(csv).toContain('billing_details_vat_number') + + const importDoc = await importCSV(csv.replace(/IT12345678901/, 'IT55555555555')) + + expect(importDoc.status).toBe('completed') + expect(importDoc.summary?.issues).toBe(0) + + const updated = await payload.findByID({ + collection: postsWithSnakeCaseFieldsSlug as CollectionSlug, + id: post.id, + }) + + expect(updated.vat_number).toBe('IT55555555555') + expect(updated.billing_details?.vat_number).toBe('IT99999999999') + }) + }) }) diff --git a/test/plugin-import-export/payload-types.ts b/test/plugin-import-export/payload-types.ts index fd249a78cd8..d69be4d353f 100644 --- a/test/plugin-import-export/payload-types.ts +++ b/test/plugin-import-export/payload-types.ts @@ -63,15 +63,15 @@ export type SupportedTimezones = | 'UTC'; /** * This interface was referenced by `Config`'s JSON-Schema - * via the `definition` "LexicalNodes_0DD453D3". + * via the `definition` "LexicalNodes_BDA6866C". */ -export type LexicalNodes_0DD453D3 = +export type LexicalNodes_BDA6866C = | SerializedTextNode | SerializedTabNode | SerializedLineBreakNode - | SerializedParagraphNode + | SerializedParagraphNode | SerializedBlockNode - | SerializedHeadingNode + | SerializedHeadingNode | SerializedUploadNode<'media', LexicalUploadFields_1AB4670B> | SerializedUploadNode<'exports'> | SerializedUploadNode<'posts-export'> @@ -88,11 +88,11 @@ export type LexicalNodes_0DD453D3 = | SerializedUploadNode<'posts-with-hooks-import'> | SerializedUploadNode<'posts-with-field-hooks-import'> | SerializedUploadNode<'posts-with-column-map-import'> - | SerializedQuoteNode - | SerializedListNode - | SerializedListItemNode - | SerializedAutoLinkNode - | SerializedLinkNode + | SerializedQuoteNode + | SerializedListNode + | SerializedListItemNode + | SerializedAutoLinkNode + | SerializedLinkNode | SerializedRelationshipNode< | 'users' | 'pages' @@ -105,6 +105,7 @@ export type LexicalNodes_0DD453D3 = | 'posts-with-hooks' | 'posts-with-field-hooks' | 'posts-with-column-map' + | 'posts-with-snake-case-fields' | 'custom-id-pages' | 'payload-kv' | 'payload-jobs' @@ -130,6 +131,7 @@ export interface Config { 'posts-with-hooks': PostsWithHook; 'posts-with-field-hooks': PostsWithFieldHook; 'posts-with-column-map': PostsWithColumnMap; + 'posts-with-snake-case-fields': PostsWithSnakeCaseField; media: Media; 'custom-id-pages': CustomIdPage; exports: Export; @@ -166,6 +168,7 @@ export interface Config { 'posts-with-hooks': PostsWithHooksSelect | PostsWithHooksSelect; 'posts-with-field-hooks': PostsWithFieldHooksSelect | PostsWithFieldHooksSelect; 'posts-with-column-map': PostsWithColumnMapSelect | PostsWithColumnMapSelect; + 'posts-with-snake-case-fields': PostsWithSnakeCaseFieldsSelect | PostsWithSnakeCaseFieldsSelect; media: MediaSelect | MediaSelect; 'custom-id-pages': CustomIdPagesSelect | CustomIdPagesSelect; exports: ExportsSelect | ExportsSelect; @@ -198,11 +201,17 @@ export interface Config { | null | ('en' | 'es' | 'de' | 'he') | ('en' | 'es' | 'de' | 'he')[]; - globals: {}; - globalsSelect: {}; + globals: { + 'payload-jobs-stats': PayloadJobsStat; + }; + globalsSelect: { + 'payload-jobs-stats': PayloadJobsStatsSelect | PayloadJobsStatsSelect; + }; locale: 'en' | 'es' | 'de' | 'he'; widgets: { collections: CollectionsWidget; + 'collection-query': CollectionQueryWidget; + activity: ActivityWidget; }; user: User; jobs: { @@ -314,7 +323,7 @@ export interface Page { | number | boolean | null; - richTextField?: LexicalRichText | null; + richTextField?: LexicalRichText | null; relationship?: (string | null) | User; excerpt?: string | null; /** @@ -382,7 +391,7 @@ export interface Hero { * via the `definition` "Content". */ export interface Content { - richText?: LexicalRichText | null; + richText?: LexicalRichText | null; id?: string | null; blockName?: string | null; blockType: 'content'; @@ -395,7 +404,7 @@ export interface FaqSection { faqs?: | { question?: string | null; - answer?: LexicalRichText | null; + answer?: LexicalRichText | null; id?: string | null; }[] | null; @@ -410,7 +419,7 @@ export interface FaqSection { export interface Post { id: string; title: string; - content?: LexicalRichText | null; + content?: LexicalRichText | null; updatedAt: string; createdAt: string; _status?: ('draft' | 'published') | null; @@ -441,7 +450,7 @@ export interface Media { export interface PostsExportsOnly { id: string; title: string; - content?: LexicalRichText | null; + content?: LexicalRichText | null; updatedAt: string; createdAt: string; _status?: ('draft' | 'published') | null; @@ -453,7 +462,7 @@ export interface PostsExportsOnly { export interface PostsImportsOnly { id: string; title: string; - content?: LexicalRichText | null; + content?: LexicalRichText | null; updatedAt: string; createdAt: string; _status?: ('draft' | 'published') | null; @@ -465,7 +474,7 @@ export interface PostsImportsOnly { export interface PostsNoJobsQueue { id: string; title: string; - content?: LexicalRichText | null; + content?: LexicalRichText | null; updatedAt: string; createdAt: string; _status?: ('draft' | 'published') | null; @@ -562,6 +571,27 @@ export interface PostsWithColumnMap { updatedAt: string; createdAt: string; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "posts-with-snake-case-fields". + */ +export interface PostsWithSnakeCaseField { + id: string; + title?: string | null; + vat_number?: string | null; + billing_details?: { + vat_number?: string | null; + }; + line_items?: + | { + item_code?: string | null; + id?: string | null; + }[] + | null; + localized_note?: string | null; + updatedAt: string; + createdAt: string; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "custom-id-pages". @@ -1179,6 +1209,15 @@ export interface PayloadJob { | number | boolean | null; + meta?: + | { + [k: string]: unknown; + } + | unknown[] + | string + | number + | boolean + | null; completedAt?: string | null; totalTried?: number | null; /** @@ -1240,7 +1279,8 @@ export interface PayloadJob { taskSlug?: ('inline' | 'createCollectionExport' | 'createCollectionImport') | null; queue?: string | null; waitUntil?: string | null; - processing?: boolean | null; + processingUntil?: string | null; + processingToken?: string | null; updatedAt: string; createdAt: string; } @@ -1295,6 +1335,10 @@ export interface PayloadLockedDocument { relationTo: 'posts-with-column-map'; value: string | PostsWithColumnMap; } | null) + | ({ + relationTo: 'posts-with-snake-case-fields'; + value: string | PostsWithSnakeCaseField; + } | null) | ({ relationTo: 'media'; value: string | Media; @@ -1604,6 +1648,28 @@ export interface PostsWithColumnMapSelect { updatedAt?: T; createdAt?: T; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "posts-with-snake-case-fields_select". + */ +export interface PostsWithSnakeCaseFieldsSelect { + title?: T; + vat_number?: T; + billing_details?: + | T + | { + vat_number?: T; + }; + line_items?: + | T + | { + item_code?: T; + id?: T; + }; + localized_note?: T; + updatedAt?: T; + createdAt?: T; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "media_select". @@ -2089,6 +2155,7 @@ export interface PayloadKvSelect { export interface PayloadJobsSelect { input?: T; taskStatus?: T; + meta?: T; completedAt?: T; totalTried?: T; hasError?: T; @@ -2109,7 +2176,8 @@ export interface PayloadJobsSelect { taskSlug?: T; queue?: T; waitUntil?: T; - processing?: T; + processingUntil?: T; + processingToken?: T; updatedAt?: T; createdAt?: T; } @@ -2145,6 +2213,34 @@ export interface PayloadMigrationsSelect { updatedAt?: T; createdAt?: T; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "payload-jobs-stats". + */ +export interface PayloadJobsStat { + id: string; + stats?: + | { + [k: string]: unknown; + } + | unknown[] + | string + | number + | boolean + | null; + updatedAt?: string | null; + createdAt?: string | null; +} +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "payload-jobs-stats_select". + */ +export interface PayloadJobsStatsSelect { + stats?: T; + updatedAt?: T; + createdAt?: T; + globalType?: T; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "collections_widget". @@ -2155,6 +2251,102 @@ export interface CollectionsWidget { }; width: 'full'; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "collection-query_widget". + */ +export interface CollectionQueryWidget { + data?: { + title?: string | null; + relatedCollection: + | 'users' + | 'pages' + | 'posts' + | 'posts-exports-only' + | 'posts-imports-only' + | 'posts-no-jobs-queue' + | 'posts-with-limits' + | 'posts-with-s3' + | 'posts-with-hooks' + | 'posts-with-field-hooks' + | 'posts-with-column-map' + | 'posts-with-snake-case-fields' + | 'media' + | 'custom-id-pages' + | 'exports' + | 'posts-export' + | 'posts-no-jobs-queue-export' + | 'posts-with-s3-export' + | 'posts-with-limits-export' + | 'posts-with-hooks-export' + | 'posts-with-field-hooks-export' + | 'posts-with-column-map-export' + | 'imports' + | 'posts-import' + | 'posts-with-s3-import' + | 'posts-with-limits-import' + | 'posts-with-hooks-import' + | 'posts-with-field-hooks-import' + | 'posts-with-column-map-import' + | 'payload-jobs'; + where?: + | { + [k: string]: unknown; + } + | unknown[] + | string + | number + | boolean + | null; + sortField?: string | null; + sortDirection?: ('asc' | 'desc') | null; + limit?: number | null; + }; + width: 'x-small' | 'small' | 'medium' | 'large' | 'x-large' | 'full'; +} +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "activity_widget". + */ +export interface ActivityWidget { + data?: { + excludedCollections?: + | ( + | 'users' + | 'pages' + | 'posts' + | 'posts-exports-only' + | 'posts-imports-only' + | 'posts-no-jobs-queue' + | 'posts-with-limits' + | 'posts-with-s3' + | 'posts-with-hooks' + | 'posts-with-field-hooks' + | 'posts-with-column-map' + | 'posts-with-snake-case-fields' + | 'media' + | 'custom-id-pages' + | 'exports' + | 'posts-export' + | 'posts-no-jobs-queue-export' + | 'posts-with-s3-export' + | 'posts-with-limits-export' + | 'posts-with-hooks-export' + | 'posts-with-field-hooks-export' + | 'posts-with-column-map-export' + | 'imports' + | 'posts-import' + | 'posts-with-s3-import' + | 'posts-with-limits-import' + | 'posts-with-hooks-import' + | 'posts-with-field-hooks-import' + | 'posts-with-column-map-import' + | 'payload-jobs' + )[] + | null; + }; + width: 'x-small' | 'small' | 'medium' | 'large' | 'x-large' | 'full'; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "TaskCreateCollectionExport". @@ -2176,6 +2368,7 @@ export interface TaskCreateCollectionExport { | 'posts-with-hooks' | 'posts-with-field-hooks' | 'posts-with-column-map' + | 'posts-with-snake-case-fields' | 'media' | 'custom-id-pages' | 'exports' diff --git a/test/plugin-import-export/shared.ts b/test/plugin-import-export/shared.ts index 149e8f2d777..a7b22c13816 100644 --- a/test/plugin-import-export/shared.ts +++ b/test/plugin-import-export/shared.ts @@ -29,3 +29,5 @@ export const postsWithHooksImportSlug = 'posts-with-hooks-import' export const postsWithFieldHooksSlug = 'posts-with-field-hooks' export const postsWithColumnMapSlug = 'posts-with-column-map' + +export const postsWithSnakeCaseFieldsSlug = 'posts-with-snake-case-fields'